{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "import torch as t"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ReLU + Sigmoid + Cross Entropy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        \n",
    "        self.fc1 = nn.Linear(28*28, 300)\n",
    "        self.fc2 = nn.Linear(300, 10)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x.view(x.shape[0], -1)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.logsigmoid(self.fc2(x))\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mnist_loader import load_data_shared, vectorized_result\n",
    "training_data, validation_data, test_data = load_data_shared(filename=\"../mnist.pkl.gz\",\n",
    "                                                                     seed=666,\n",
    "                                                                     train_size=1000,\n",
    "                                                                     vali_size=0,\n",
    "                                                                     test_size=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(data, net):\n",
    "    with t.no_grad():\n",
    "        #for index in range(test_data[0].shape[0]):\n",
    "            # get the inputs\n",
    "        inputs, labels = t.Tensor(data[0]), data[1]\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        _, predicted = t.max(outputs, 1)\n",
    "\n",
    "        correct = (predicted == t.Tensor(labels)).sum().item()\n",
    "        accuracy = correct / data[0].shape[0]\n",
    "        return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit(net, criterion, optimizer):\n",
    "    loss_scores = []\n",
    "    test_scores = []\n",
    "    train_scores = []\n",
    "    for epoch in range(100):  # loop over the dataset multiple times\n",
    "\n",
    "        # get the inputs\n",
    "        inputs, labels = t.Tensor(training_data[0]), t.Tensor(training_data[1])\n",
    "        vector_labels = t.Tensor([vectorized_result(y) for y in training_data[1]])\n",
    "        # zero the parameter gradients\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        loss = criterion(outputs, labels.long())\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        # print statistics\n",
    "        loss_scores.append(loss.item())\n",
    "        train_scores.append(predict(training_data, net))\n",
    "        test_scores.append(predict(test_data, net))\n",
    "    print('Finished Training')\n",
    "    return loss_scores, train_scores, test_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "import torch.optim as optim\n",
    "net = Net()\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(net.parameters(), lr = 1e-1)\n",
    "loss_scores, train_scores, test_scores = fit(net, criterion, optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXhV1b3/8fc3EyQBMpAwJiHMMgrkMIqIWlscropSKg6oqEjVVq29re1t76+z1dZaESdERWoFB6hV68wgooAkzDLPRIYEEwKEISRZvz/OsZdiAsGcZOec83k9Tx6yz16e/d3u8GFnnbXXMuccIiIS+qK8LkBERIJDgS4iEiYU6CIiYUKBLiISJhToIiJhIsarA6elpbns7GyvDi8iEpLy8vL2OefSq9rnWaBnZ2eTm5vr1eFFREKSmW2vbp+6XEREwoQCXUQkTCjQRUTChAJdRCRMKNBFRMKEAl1EJEwo0EVEwkTIBfrWfaU8/P565q0v4MDR416XIyLSYHj2YNE3tfqLEh6fu4lKB2bQtWVTctql/PsrKzUBM/O6TBGRemdeLXDh8/ncN31StPRYOct37id3WzG524tYvmM/B4+VA5DWJI4+mf5w75eVTO+MZOLjooNZuoiIZ8wszznnq2pfyN2hAyQ2iuGcTmmc0ykNgIpKx8aCg+RuK2bpjmKW7djPh2v3AhAdZXRr3ZS+mSn0a5dM38wU2jXXXbyIhJ+QvEOviaLSMpYFwn3pjmJW7NxPaVkFACkJsfTN8t/B981K4ezMZJo0Csl/20QkwoTdHXpNpCbGcWG3llzYrSXwf3fxS7fv9wf9zv3MWVcA/F9ffN+sFPpmJdMvK4UOaYlERekuXkRCR9jeoddEyeHjLM/fz9Lt/q6a5Tv3c/Covy8+KT723+Ge00538SLSMETkHXpNJCXEcl6XdM7r4p9auLLSsWXfIZZu93fTLN1RzLz1hQBEGXRt1Yx+Wcn4slPIyUolMzVeffEi0mBE9B16TZQcOc7ynfvJ217M0u3+u/hDgRE16U0b4QsMl/Rlp9KjTTNio0NuaL+IhBDdoddCUvx/3sVXVDo27D1I3vZi8rb7h02+s3oPAI1jo+ibmUL/bH/A92uXom4aEak3ukMPgr0Hjv57THzutmI+31VCpfMPmezeuhkD2qf6v7JTSUmM87pcEQlhp7pDP22gm1kmMA1oBVQCk51zj57U5jrgp4HNQ8D3nXMrTvW+4RToJzt0rJyl24tZsq2IxVuLWL5zP2XllYB/NM2gDqkM7NCcge1Tad6kkcfVikgoqW2gtwZaO+eWmllTIA+40jm35oQ2Q4C1zrliM7sY+JVzbuCp3jecA/1kx8orWJlfwmdbi1i05UtytxVz5Lh/THzXlk0Z3LE5gzs2Z1CH5iTFx3pcrYg0ZLUK9Cre7J/AJOfcB9XsTwFWO+fanup9IinQT3a8opKV+SUs2vIli7Z8yZJtRRw9XkmUQc+2SQzpmMbQTmn4slNoHKtpC0Tk/wQt0M0sG5gP9HTOHaimzY+Bs5xzt57qvSI50E92rLyCFTtL+GTTPj7dvI9lO/ZTXumIi4mif3YKQzulc27nNLq3bqaHnUQiXFAC3cyaAB8Bv3fOzaqmzfnAE8BQ59yXVewfD4wHyMrKytm+fXvNziDClB4r57OtRSzYtI8FG/exfu9BAJonxjG0cxrDOqczrEs66U3V/y4SaWod6GYWC7wFvOec+0s1bXoD/wAuds5tON176g695vYeOMqCjfv4eGMhH2/cx5elZQD0bNuM4V1acP5Z6fTJTCFad+8iYa+2H4oa8AJQ5Jy7p5o2WcAcYKxz7tOaFKVA/2YqKx2f7zrARxsKmLe+kKU7iql0kBx46vWCs1owvEsLkhL04apIOKptoA8FPgZW4R+2CPBzIAvAOfeUmU0Brga+6kMpr+6AX1GgB0fJ4ePM31jI3PUFfLS+kC9Ly4iOMnztUvhWt5Zc1L0l2WmJXpcpIkES1FEuwaJAD76KSseK/P3MXruX2WsLWLfH3/feuUUTvt2jJd/u3oreGUmaf0YkhCnQI9TOosN8uHYvH6zZy+KtRVRUOlonNeY7PVoxomcr+menqt9dJMQo0IX9h8uYvbaAdz/fw/wNhRwrryStSRzf7tGKS3u1ZmD7VGI0sZhIg6dAl/9QeqyceesLeXv1buauK+BwWQVpTeIY0bMVl/Vuw4DsVI13F2mgFOhSrSNlFcxbX8Bbq3YzZ20BR45X0KpZYy7r3Zor+rSlZ9tm6nMXaUAU6FIjh8vK+XBtAW+u2MVH6wspq6ikY3oiV/Zpy5V925KZmuB1iSIRT4EuZ6zk8HHeXr2bfyz7gs+2FgEwsH0qV/fL4OJerWjaWOPcRbygQJdayS8+zOvLvmDm0i/Yuq+U+NhoLu7VitG+TAa2T1WXjEg9UqBLUDjnWLZzP6/m5vPmil0cOlZOdvMEvtc/i6tz2tKiaWOvSxQJewp0CbojZRW8s3o3M5bs5LOtRcREGRd1b8l1A9sxpGNzjZIRqSMKdKlTmwsP8fKSnbyau5Piw8fJbp7AdQPbMdqXqTllRIJMgS714ujxCt5dvYcXF20nd3sxjWOjGNm3LTcOyeasVs28Lk8kLCjQpd59vquEvy3czuvLv+Do8UqGdGzOLUPbc37XFuqOEakFBbp4Zv/hMmYs2ckLn25jd8lR2qclcsvQ9ozKydDyeiLfgAJdPHe8opJ3V+9hysdbWJFfQvPEOMYOzubGIe1ITojzujyRkKFAlwbDOcfirUVMnr+FOesKSIyL5rpB7bh1aHtaNNOwR5HTUaBLg7RuzwGenLeZN1fsIiY6ijH9M5kwvCOtk+K9Lk2kwVKgS4O2/ctSnpy3mdfy8okyY3T/DO48v5OCXaQKCnQJCfnFh3li3mZezd2JmXH9wHZ8f3hH0ps28ro0kQbjVIF+2hUNzCzTzOaa2Voz+9zM7q6ijZnZRDPbZGYrzaxfMAqXyJKRksAfRvZizn3DubJPG15YuI1hD83l4ffXc/Doca/LE2nwarJETTlwn3OuGzAIuNPMup/U5mKgc+BrPPBkUKuUiJKZmsBDo87mg3uHcUG3Fjw2ZxPn/Wkezy7YSll55enfQCRCnTbQnXO7nXNLA98fBNYCbU9qdgUwzfktApLNrHXQq5WI0iG9CY9f24837xpKt9ZN+e1ba7jokY94d/UevOoqFGnIzmgRSTPLBvoCi0/a1RbYecJ2Pl8PfcxsvJnlmlluYWHhmVUqEatXRhJ/v3UQU2/uT1x0FBNezON7kxfx+a4Sr0sTaVBqHOhm1gSYCdzjnDtw8u4q/pOv3UI55yY753zOOV96evqZVSoRb3jXFrxz97n87sqebCo4xH89toBfvL6K/YfLvC5NpEGoUaCbWSz+MP+7c25WFU3ygcwTtjOAXbUvT+Q/xURHcf2gdsy9bzhjB2fz0uIdDP/zPGZ8toPKSnXDSGSrySgXA54F1jrn/lJNszeAsYHRLoOAEufc7iDWKfIfkhJi+dXlPXj77nPp0qIp989axeinF7Juz8m/PIpEjtOOQzezocDHwCrgqyEGPweyAJxzTwVCfxIwAjgM3OycO+Ugc41Dl2BxzvFqXj4PvL2Wg0fLGT+sAz+8sLMm/5KwpAeLJCIUlZbx+3+tZebSfDqkJ/LQ1b3xZad6XZZIUNXqwSKRUJGaGMfDo89m2rgBHDteyXefXsiv3vicI2UVXpcmUi8U6BJ2hnVJ5/17hzF2UDumfrqNSyd+zNIdxV6XJVLnFOgSlhIbxfDrK3ry91sHcvR4BaOe/JSH31/P8Qo9aSrhS4EuYe2cTmm8e+8wRvbN4LE5m/juUwvZ/mWp12WJ1AkFuoS9Zo1jeXj02Uy6ti+bCw9xyaMfM2tpvtdliQSdAl0ixmW92/DuPcPo0SaJH72ygp+8tkIfmEpYUaBLRGmbHM9Ltw3krvM78UpuPlc+/gmbCw95XZZIUCjQJeLEREfx4+905YVxAyg8dIwrJn3Cu6v3eF2WSK0p0CVindclnX/9cCgdWzRhwot5/Om9dVRoPhgJYQp0iWitk+J55fZBjBmQyeNzN3Pz1CWUHNHqSBKaFOgS8RrFRPPAVb154KpeLNy8j5FPfMLWfRraKKFHgS4SMGZAFi/eMpDi0jKufPwTPtm0z+uSRM6IAl3kBAM7NOeNu4bSslkjbnzuM17J3Xn6/0ikgVCgi5wkMzWBmd8fwuCOzfnJayv5ywcbtIaphAQFukgVmjaO5bmb+jMqJ4OJszfy41dXah4YafBivC5ApKGKjY7iT6N6k5ESz18/3EhR6TGeuC6H+DgtnCENk+7QRU7BzLjnW134/ciezNtQyPXPLtai1NJg1WRN0efMrMDMVlezP8nM3jSzFWb2uZndHPwyRbx13cB2PH5tP1bll/C9pxdRcOCo1yWJfE1N7tCn4l8rtDp3Amucc2cDw4GHzSyu9qWJNCyX9GrN1Jv7s7P4MN+bvIhd+494XZLIfzhtoDvn5gNFp2oCNA0sFN0k0LY8OOWJNCxDOqXxt1sGsO/gMc2tLg1OMPrQJwHdgF3AKuBu51yVwwHMbLyZ5ZpZbmFhYRAOLVL/ctql8tJtgygtK2f00wvZotkapYEIRqB/B1gOtAH6AJPMrFlVDZ1zk51zPuecLz09PQiHFvFGr4wkXh4/mPIKx5hnFmmqAGkQghHoNwOznN8mYCtwVhDeV6RB69qqKS/dNojyCsc1kxcq1MVzwQj0HcCFAGbWEugKbAnC+4o0eF+F+vEKx5jJi9SnLp6qybDF6cBCoKuZ5ZvZLWY2wcwmBJr8FhhiZquA2cBPnXOa1Ugihj/UB3K0vIJrn1ms0S/iGfNqjgqfz+dyc3M9ObZIXViVX8K1zywivWkjXr59MOlNG3ldkoQhM8tzzvmq2qcnRUWCpFdGEs/f3J/dJUe5QU+UigcU6CJB5MtOZcqNPrYUljJu6hIOl+mRDKk/CnSRIDunUxoTx/Rh+c793PH3pZqlUeqNAl2kDozo2Zrfj+zFvPWF/PerK6jU4tNSDzR9rkgdGTMgi6LSMv703nrSmjTiF5d197okCXMKdJE6dMfwjhQePMaUBVtpkxzPuKHtvS5JwpgCXaQOmRm/vKw7u0uO8Nt/raFVUmMu6dXa67IkTKkPXaSORUcZj17Tl35ZKdzz8nKWbDvV5KUi35wCXaQeNI6NZspYHxnJ8Yyflss2zfsidUCBLlJPUhLjeO6m/gCMm7pEDx5J0CnQRepRdloik8f6yC8+woQX8ygr1xh1CR4Fukg965+dykOjerNoSxH/849VeDWfkoQfjXIR8cCVfduyZV8pE2dvpEvLptw2rIPXJUkY0B26iEfuubAzF/dsxR/eWcucdXu9LkfCgAJdxCNRUcbDo8+me+tm/HD6ctbvOeh1SRLiFOgiHkqIi2HKjT7i46K5bVquRr5IrSjQRTzWOimep67PYU/JUe56aRnlmp1RvqGaLEH3nJkVmNnqU7QZbmbLzexzM/souCWKhL+cdin8bmRPFmzaxwPvrPO6HAlRNblDnwqMqG6nmSUDTwCXO+d6AN8NTmkikWW0L5ObhmTz7IKtvJaX73U5EoJOG+jOufnAqSafuBaY5ZzbEWhfEKTaRCLO/1zajSEdm/Pzf6xiZf5+r8uREBOMPvQuQIqZzTOzPDMbW11DMxtvZrlmlltYWBiEQ4uEl9joKCZd24/0Jo24/W957Dt0zOuSJIQEI9BjgBzgUuA7wC/NrEtVDZ1zk51zPuecLz09PQiHFgk/qYlxPH1DDkWlZdypJezkDAQj0POBd51zpc65fcB84OwgvK9IxOrZNok/Xt2LxVuL+P2/1npdjoSIYAT6P4FzzSzGzBKAgYB+AkVqaWTfDMad056pn27j9WVfeF2OhIDTzuViZtOB4UCameUD/w+IBXDOPeWcW2tm7wIrgUpginOu2iGOIlJzP7vkLFbvKuH+WSvp0rIp3ds087okacDMq5nefD6fy83N9eTYIqGk8OAxLnvsY+JionjzrqEkJ8R5XZJ4yMzynHO+qvbpSVGRBi69aSOeDDxJeveM5VRUarpdqZoCXSQE9MtK4VeX9+CjDYU8Onuj1+VIA6VAFwkR1w7IYlROBhNnb2T2Wk23K1+nQBcJEWbG767sSY82zbjn5eVaaFq+RoEuEkIax0bz1PU5RJkx4cU8jpRVeF2SNCAKdJEQk5mawKPX9GH93oP8XGuSygkU6CIhaHjXFtxzYRf+sewLXly03etypIFQoIuEqB9c0IkLzmrBb95aQ972Yq/LkQZAgS4SoqKijEdG96F1Ujx3/D2PwoOamTHSKdBFQlhSQixPXZ9DyZHj3PXSUi1fF+EU6CIhrnubZjxwlX9mxj9q+bqIpkAXCQMj+2YwdnA7pizYypsrdnldjnhEgS4SJn5xaXdy2qXw05krWb/noNfliAcU6CJhIi4miieu60dCXAwTXsyj5Mhxr0uSeqZAFwkjLZs15onr+rGz6DD3vbKcSs3MGFEU6CJhZkD7VH5xaTc+XFvApLmbvC5H6pECXSQM3Tgkm5F92/LIhxuYs04zM0aK0wa6mT1nZgVmdspl5cysv5lVmNmo4JUnIt+EmfGHkb3o1qoZd89YzlbNzBgRanKHPhUYcaoGZhYNPAi8F4SaRCQI4uOiefqGHGKijPHTcjl0rNzrkqSOnTbQnXPzgaLTNPsBMBMoCEZRIhIcmakJTLq2H5sLD/Hfr67QzIxhrtZ96GbWFhgJPFWDtuPNLNfMcgsLC2t7aBGpgXM6pfGzi7vxzuo9PK4PScNaMD4U/SvwU+fcaWfad85Nds75nHO+9PT0IBxaRGri1nPbc2WfNvz5/Q18uEYfkoarYAS6D5hhZtuAUcATZnZlEN5XRILEzPjj1b3p1TaJe15ezqYCPUkajmod6M659s65bOdcNvAacIdz7vVaVyYiQdU41v8haePYKG59IZeSw3qSNNzUZNjidGAh0NXM8s3sFjObYGYT6r48EQmmNsnxPHV9Dl/sP8Jd0zXdbrgxrz719vl8Ljc315Nji0S6V5bs5CczV3LTkGx+dXkPr8uRM2Bmec45X1X7Yuq7GBHx3uj+mazfe5BnF2yla6umjBmQ5XVJEgR69F8kQv3s4rM4r0s6v3x9NYu2fOl1ORIECnSRCBUTHcXEMX1p1zyBCS/msU3TA4Q8BbpIBEuKj+W5m/pjwLgXlmjkS4hToItEuHbNE3n6Bh87iw5zx0t5HNfIl5ClQBcRBrRP5YGrevPJpi/55eurNedLiNIoFxEBYFROBtv2lTJp7iaymidwx/BOXpckZ0iBLiL/dt+3u7Cj6DAPvbuezJQE/uvsNl6XJGdAgS4i/2Zm/Om7vdlTcpT7Xl1Bq6TG9M9O9bosqSH1oYvIf2gU45/zJSM5nltfyGVTwSGvS5IaUqCLyNekJMbxwrgBxEYbNz73GQUHjnpdktSAAl1EqpSZmsDzNw2g+HAZNz2/hINHNUa9oVOgi0i1emUk8fh1/Vi/9yATXszjWPlp17ERDynQReSUzu/aggev9o9Rv++VFVRUaox6Q6VRLiJyWqNyMvjy0DEeeGcdqYlx/PryHpiZ12XJSRToIlIjt5/XkS9Ly5g8fwvNExtx97c6e12SnESBLiI1dv+IsygqLeORDzfQtHEM44a297okOUFNlqB7zswKzGx1NfuvM7OVga9Pzezs4JcpIg1BVJTxx6t68Z0eLfnNW2t4NXen1yXJCWryoehUYMQp9m8FznPO9QZ+C0wOQl0i0kB9NY/6uZ3T+OnMlbyzarfXJUnAaQPdOTcfKDrF/k+dc8WBzUVARpBqE5EG6qunSftmpfDDGcuYs26v1yUJwR+2eAvwTnU7zWy8meWaWW5hYWGQDy0i9SkhLobnb+5Pt9bNmPDiUj7eqL/TXgtaoJvZ+fgD/afVtXHOTXbO+ZxzvvT09GAdWkQ80qxxLNPGDaBDWiK3TcvV2qQeC0qgm1lvYApwhXNOV1QkgiQnxPHirQPJSElg3NQlfLa12h5aqWO1DnQzywJmATc45zbUviQRCTVpTRrx0q0DaZXUmJue/0yh7pGaDFucDiwEuppZvpndYmYTzGxCoMn/As2BJ8xsuZnl1mG9ItJAtWjWmBm3Dfp3qC/ZplCvb+bV2oE+n8/l5ir7RcJNwYGjXDN5EXsOHOXZG/szuGNzr0sKK2aW55zzVbVPk3OJSFC1aNaYGeMH0TY5npue/4z5GzT6pb4o0EUk6L4K9Q7pTbj1hVyNU68nCnQRqRPNmzRi+m0DOat1U8ZPy+Otlbu8LinsKdBFpM58NaSxb1YyP5i+jBmf7fC6pLCmQBeROuV/+Gggwzqnc/+sVTwzf4vXJYUtBbqI1Ln4uGieGevj0l6t+f3ba/njO+vwaoRdONN86CJSL+Ji/LM0JifE8tRHm/0rIF3Vi5ho3VcGiwJdROpNdJTxuyt7ktakEY/O3khRaRmPXduXhDhFUTDon0YRqVdmxr0XdeG3V/RgzvoCxjyzmH2HjnldVlhQoIuIJ24YnM3T1+ewfs8BrnriU7YUHvK6pJCnQBcRz3y7Ryum3zaI0mPlXP3kp5rUq5YU6CLiqb5ZKcy6YwgpCXFcP2Ux/1iW73VJIUuBLiKea9c8kVl3DCGnXQr3vryCh99fT2WlhjWeKQW6iDQIyQlxvDBuAKN9GTw2ZxN3vrSUw2XlXpcVUhToItJgxMVE8eDVvfnFpd147/M9XP3kQvKLD3tdVshQoItIg2Jm3HpuB567qT/5xYe5YtInWqu0hhToItIgDe/agtfvPIekhFium7KYZxds1XQBp1GTJeieM7MCM1tdzX4zs4lmtsnMVppZv+CXKSKRqGN6E/555zlceFYLfvvWGu55ebn61U+hJnfoU4ERp9h/MdA58DUeeLL2ZYmI+DVtHMtT1+fw39/pyhsrdjHy8U/ZrIeQqnTaQHfOzQdONdr/CmCa81sEJJtZ62AVKCISFWXceX4nXrh5AIWHjnH5Ywu0YEYVgtGH3hbYecJ2fuC1rzGz8WaWa2a5hYVaZ1BEzsywLum89YOhdG3VlLteWsb//nM1R49XeF1WgxGMQLcqXqvykwvn3GTnnM8550tPTw/CoUUk0rRJjmfG+MHcOrQ90xZu1zwwJwhGoOcDmSdsZwD6XUhE6kxcTBS/uKw7z97oY1fJES57bAEz8/IjfhRMMAL9DWBsYLTLIKDEObc7CO8rInJKF3ZryTt3n0vPtknc9+oKfjhjOSVHjntdlmdqMmxxOrAQ6Gpm+WZ2i5lNMLMJgSZvA1uATcAzwB11Vq2IyElaJ8Uz/bZB/PjbXXh71W4uefTjiJ210bz6FcXn87nc3FxPji0i4WnpjmLumbGcncWHGT+sAz+6qAuNYqK9LiuozCzPOeerap+eFBWRsNEvK4W37z6Xa/pn8vRHW7hi0ies2XXA67LqjQJdRMJKk0YxPHBVb567yce+Q2VcPmkBj364keMVlV6XVucU6CISli44qyUf3DuMS3q15pEPN0TE3boCXUTCVkpiHBPH9OWp63MoOHiUyyct4E/vrQvbh5EU6CIS9kb0bMUH957H5X3a8PjczVwyMTxHwijQRSQipCTG8ZfRfZg2bgBl5ZWMfnoh989cyf7DZV6XFjQKdBGJKMO6pPP+vcO4/bwOvJqXz4UPf8SspeHxlKkCXUQiTkJcDD+7uBtv/WAoWc0T+NErK/je04tYtye0PzRVoItIxOrWuhkzJwzhj1f1YmPBQS6duIDfvLkmZKcPUKCLSESLijKuGZDFnPuG873+mTz/6VYu+PM8Zny2g4rK0OqGUaCLiOD/0PQPI3vx5l1DaZ+WyP2zVnH5pAUhtUC1Al1E5AQ92ybx6oTBPHpNH4pKy7hm8iJu/1suW/eVel3aaWlyLhGRahwpq+DZBVt4Yt5mjldUct3Advzggk40b9LIs5pONTmXAl1E5DQKDh7lrx9u5OUlO4mPjeb2YR0YN7Q9iY1i6r0WBbqISBBsKjjIg++u54M1e0lrEsdd53dizMCsep2iV4EuIhJEeduLeOjd9SzeWkTb5HjuvrAzI/u1JTa67j+W1HzoIiJBlNMulRnjBzFt3ACaN4njJzNX8q2/+J84Lfdwmt4aBbqZjTCz9Wa2yczur2J/lpnNNbNlZrbSzC4JfqkiIg2HmTGsSzr/vPMcpoz1kRgXw49eWcFFj8zntTxvgv20XS5mFg1sAC4C8oElwBjn3JoT2kwGljnnnjSz7sDbzrnsU72vulxEJJxUVjreX7OXibM3smb3AbJSE/j+8I5c1a9tUPvYa9vlMgDY5Jzb4pwrA2YAV5zUxgHNAt8nAbu+abEiIqEoKsoY0bMV//rhUJ4Z6yMpPpafzVrFeQ/N49kFWzlcVl7nNdTkDn0UMMI5d2tg+wZgoHPurhPatAbeB1KAROBbzrm8Kt5rPDAeICsrK2f79u3BOg8RkQbFOcfHG/fx+NxNLN5aRHJCLGMHZ3Pj4Ha1Gsde2zt0q6rWk7bHAFOdcxnAJcDfzOxr7+2cm+yc8znnfOnp6TU4tIhIaPqqj/3l2wcz8/uD6Z+dysTZGznnwTlM+XhLnRyzJqPi84HME7Yz+HqXyi3ACADn3EIzawykAQXBKFJEJJTltEvlmbGpbCo4xDPzt5CREl8nx6lJoC8BOptZe+AL4Brg2pPa7AAuBKaaWTegMVAYzEJFREJdpxZNeHBU7zp7/9N2uTjnyoG7gPeAtcArzrnPzew3ZnZ5oNl9wG1mtgKYDtzkwmH5DxGREFKjiQicc28Db5/02v+e8P0a4JzgliYiImdCT4qKiIQJBbqISJhQoIuIhAkFuohImFCgi4iECQW6iEiY8GyBCzMrBL7pZC5pwL4glhMqIvG8I/GcITLPOxLPGc78vNs556qcO8WzQK8NM8utbnKacBaJ5x2J5wyRed6ReM4Q3PNWl4uISJhQoIuIhIlQDfTJXhfgkUg870g8Z4jM847Ec4YgnndI9qGLiMjXheoduoiInESBLiISJkIu0M1shJmtN7NNZna/1/XUBTPLNLO5ZrbWzD43s7sDr2Vke1EAAANoSURBVKea2QdmtjHwZ4rXtdYFM4s2s2Vm9lZgu72ZLQ6c98tmFud1jcFkZslm9pqZrQtc88GRcK3N7N7Az/dqM5tuZo3D8Vqb2XNmVmBmq094rcrra34TA/m20sz6ncmxQirQzSwaeBy4GOgOjDGz7t5WVSfKgfucc92AQcCdgfO8H5jtnOsMzA5sh6O78S+m8pUHgUcC512Mf8nDcPIo8K5z7izgbPznHtbX2szaAj8EfM65nkA0/tXQwvFaTyWwROcJqru+FwOdA1/jgSfP5EAhFejAAGCTc26Lc64MmAFc4XFNQeec2+2cWxr4/iD+v+Bt8Z/rC4FmLwBXelNh3TGzDOBSYEpg24ALgNcCTcLqvM2sGTAMeBbAOVfmnNtPBFxr/AvsxJtZDJAA7CYMr7Vzbj5QdNLL1V3fK4Bpzm8RkGxmrWt6rFAL9LbAzhO28wOvhS0zywb6AouBls653eAPfaCFd5XVmb8CPwEqA9vNgf2BpRAh/K55B/zr7z4f6GaaYmaJhPm1ds59AfwZ/3rEu4ESII/wvtYnqu761irjQi3QrYrXwnbcpZk1AWYC9zjnDnhdT10zs8uAAudc3okvV9E0nK55DNAPeNI51xcoJcy6V6oS6DO+AmgPtAES8Xc3nCycrnVN1OrnPdQCPR/IPGE7A9jlUS11ysxi8Yf5351zswIv7/3q16/AnwVe1VdHzgEuN7Nt+LvTLsB/x54c+LUcwu+a5wP5zrnFge3X8Ad8uF/rbwFbnXOFzrnjwCxgCOF9rU9U3fWtVcaFWqAvAToHPgmPw/8hyhse1xR0gX7jZ4G1zrm/nLDrDeDGwPc3Av+s79rqknPuZ865DOdcNv5rO8c5dx0wFxgVaBZW5+2c2wPsNLOugZcuBNYQ5tcaf1fLIDNLCPy8f3XeYXutT1Ld9X0DGBsY7TIIKPmqa6ZGnHMh9QVcAmwANgP/43U9dXSOQ/H/mrUSWB74ugR/f/JsYGPgz1Sva63D/wfDgbcC33cAPgM2Aa8CjbyuL8jn2gfIDVzv14GUSLjWwK+BdcBq4G9Ao3C81sB0/J8THMd/B35LddcXf5fL44F8W4V/FFCNj6VH/0VEwkSodbmIiEg1FOgiImFCgS4iEiYU6CIiYUKBLiISJhToIiJhQoEuIhIm/j8XyM8dK/CSYQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(loss_scores)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD7CAYAAAB68m/qAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd3xUdb7/8dcnnSS0FGoICRCaFMFIFQXFFUTAtggWcEXZ3bvuqmvdn2XVe/darlssXPayWABB7AoK6kpRpCYUI50QQggB0khIb/P9/XEGCWGSTGCSSWY+z8cjDzjnfGfmczzw9vCd7/l+xRiDUkqpls/H3QUopZRyDQ10pZTyEBroSinlITTQlVLKQ2igK6WUh9BAV0opD+FUoIvIBBHZLyLJIvKEg+PdRWS1iCSJyDoRiXJ9qUoppeoi9Y1DFxFf4ABwLZAOJAAzjDF7qrX5EPjCGLNQRK4GfmWMuavxylZKKVWTnxNthgHJxpgUABFZBkwF9lRr0x94yP77tcBn9b1pRESEiYmJaVCxSinl7bZt25ZtjIl0dMyZQO8KHK22nQ4Mr9HmR+AW4FXgJqC1iIQbY3Jqe9OYmBgSExOd+HillFJniMiR2o4504cuDvbV7Kd5BLhKRHYAVwHHgEoHhcwRkUQRSczKynLio5VSSjnLmUBPB7pV244CMqo3MMZkGGNuNsYMAZ6078uv+UbGmPnGmHhjTHxkpMN/MSillLpAzgR6AhAnIrEiEgBMB5ZXbyAiESJy5r3+BLzl2jKVUkrVp95AN8ZUAvcDXwN7gQ+MMbtF5HkRmWJvNhbYLyIHgI7AXxqpXqWUUrWod9hiY4mPjzf6pahSSjWMiGwzxsQ7OqZPiiqllIfQQFdKKQ/hzDh0pZRSdTDGkFVYRlpOMVU2qxu7ymY4VVxBTlEZuUXl2Gxnu7ev6deRwd3aubwODXSllLoAR3OL+fKn46zdl8mBkwWcKq6os71Ue6KnQ5sgDXSllHKXiiobSen5bDqUzZp9mWxPywNgQNc2TBjQiT4dWxMTEUKAr9WTLSKEhQQQHhpA++AAfH0cPaPpWhroSinlgM1m2HP8NJsO5bDxUDZbD+dSVF4FWCH++IS+3DCoM93Cgt1c6Vka6EopBZRWVJGcWcj2tFNsTM5h8+Ec8uzdKD0iQ7hxSFdG94pgRI9wwkIC3FytYxroSimvY4whLbeYjYdy2Hgoh13H8jmSU8SZ7y27tg3imY6bGFO+nrat/K1ulDwg0f5zsUb+Dvpe74I3OpcGulLKo5wJ602Hcth6OJfTpWfmCTScLq0kp7CM7MJy8kusu+8OrQMZGt2eKYO70KdTawZ0CqHblueQxAXQ4RLwbe++k2kgDXSlVNOx2aDwxAW9tLSiioLSyp+nehWgdZAfQf6+GGPYf7KQf+89wZq9mWTklQIQHhpAeLXukfaBfvSKCKBdtwBiI9sS3z2MmPBg5MwQlKoy+PL3kPxvGP0gXPNn8Gk5j+tooCulmkbBCXhvOmTsuKCXB9l/HBGgr/3n91RrWAmcN+9rPcQXJr8Kl919AVW6lwa6UsoljDEUlVf93KWRU1hGTpH1a1hhMlP3PEhgxWkOXPII6aUBnMgvJTWniOLyKlr5+xAdFkJZRRWF5VWUVVYREuBHaKAvIYH+hAT6EhroR3CALz72u2mbMZRUVFFYWklJRRUx4SFc0qUtIYG+F3cinQdDlyEu+C/S9DTQlWrOUtbVfkfbfTR0G+byj8wpLGPVrhOs259FWWXVz/vbtPInIiSAsJBAAvx8wBhislZDbgrZhWVkF5RRVmk77/0CqWCy30oKCGJa+VPs3haDr48QEx7M4L7tmDSoM2PiIq33VBdFA12p5urELlh8E5jzQxKwugZu+NtFdw3YbIZ9JwrYeCib7w5ksfFQDlU2Q/fw4J/7n20Gjp0qIbuwjNOllfhSxXN+7zDRb3W1egD/Wj6j86UUT3qLlyQCXx+hR2QIgX4XeSetzqOBrlRzZAx89QQEtYP/2AxBbc49XlECn8yBFQ9Abgpc82yDv7w7mlvMok2pfLz9GLlF5QDERoQw58oeTB7UhX6dW5/9srD6Rxfn4fPRPfimrKZy1IP4jn2Ms+vbOObjF0RHETo2qELVUBroSrlbRSl8+mvoewMM+qW1b8/nkLoeJv0VWjuIQf9WMGMZfPU4bHjVau8fcn67sFiY/ComOJzconL2nyzgwIkCfkjOZvW+TMb5/MgnoZ/TPtJGcKAv/j4+kIL1Uwv/4mwozoEpr+M3dKZL/hMo19BAV8rd9q6APZ9ZPznJMPoB+OZp6DgALvtV7a/z9YPrX4FOA+Hgv3/eXVppI7eonNzCMuL2fUP2vlHMrnyMfZWdf24TERrI/L47GZ/6VyQ0FiL7Ol9vRC+Inw09rrqQs1WNSFcsUsrdFk6GU6kQMwZ2LoGwnpB7CO7+EmKucOot0k8V82XScVYkZbDr2GkA2gT5cXOH4zyc8yx+VLLhkudoH9WbmPBgwg5+jGyeC3HXwa1vQWBoI56gcqW6Vixy6g5dRCYArwK+wAJjzIs1jkcDC4F29jZPGGNWXlTVSnmD3MNw+HsY9yRc+ajVRbLmv6D/jfWGeZXN8O89J1m4MZVNKTkADO7Wjicm9uWKXhH069zGmuHv1DhYOo3xSX+EpGpvMOzXMOEF8NEvJz1FvYEuIr7AXOBaIB1IEJHlxpg91Zo9hbV49DwR6Q+sBGIaoV6lPMvOJYDApbdbE2Zf+Sj0nmDdpddQXmnjUFYhB04WsPd4ASt+zOBYXgld27XikV/0ZsrgrkSHO5j5r313uPdb638cNvswxJAI6D6qcc9NNTln7tCHAcnGmBQAEVkGTAWqB7oBznwN3xbIcGWRSnkkWxXsWAK9xkPbqLP7Ow08p5kxhhVJx/nLl3s4eboMAD8fIT6mPU/f0J/x/Trg51vPCJfA1tB3kqvPQDUzzgR6V+Bote10YHiNNs8C34jI74EQYLyjNxKROcAcgOjo6IbWqpRnSV4NBRkw8cXam2QW8PRnu9mUksPArm35f9f3o2+nNsRGhOiDOOo8zgS6o2U2an6TOgN4xxjzVxEZCSwWkQHGnPtEhDFmPjAfrC9FL6RgpTzGjkUQHAG9Jzo8/FN6PrfN34S/rw//deMAZgyLbpJVb1TL5UygpwPdqm1HcX6XymxgAoAxZpOIBAERQKYrilTK4xRmwf5VMPw34Hf+YglHcor41TtbaR8cwMe/HUWntrVNS6XUWc78my0BiBORWBEJAKYDy2u0SQOuARCRflhznWW5slClPErSMrBVwpC7zjuUXVjGrLe2UmkzLJo9TMNcOa3eQDfGVAL3A18De7FGs+wWkedFZIq92cPAfSLyI/AecLdx1wB3pZo7Y2D7Ioi6HDqc+0BPfnEF97yTwInTpbw563J6Rur4cOU8p8ah28eUr6yx75lqv98DjHZtaUp5qKNbIfsATH7tnN1ZBWXc9eYWUrKKmHfnUC7r3nJWylHNgz76r1RT27HImndlwM0/78rIK+HOBVvIyC9hwax4ruwd6cYCVUulga5UUyorgF2fWmEe2BqAXcfy+fXibZwuqWDx7OFcHhPm5iJVS6UDWZVqSrs+gYoisM9S+H5CGjfP24jNGN6bM0LDXF0UvUNXqintWAwRfSjrNJSnP/qRDxLTuaJXBK9Ov5Tw0EB3V6daOA10pZpK9kFIT6B47PPc/WYCW1Nz+f3VvXhwfG99YEi5hAa6Uk3l8HcA3Le1AztP5/H6jCFMHtzFzUUpT6KBrlQTyd23nirasackjKX3Xk689pcrF9NAV6oJbDqUQ3TyBg749efT315BTISD5eKUukg6ykWpRrZ2XyaPv72KrpJF/JgJGuaq0WigK9WIPtmezn2LEpnULg2A1nFj3FyR8mTa5aJUIzDG8PdvD/La6oOM6hnOg51yISn4vMUrlHIlvUNXysXKKqt46P2dvLb6IL+8LIp3fjWMwGNbICoefP3dXZ7yYBroSrlQcXkl9y5M5LOdGTx6XR9evnUQAVVFcHIXRI90d3nKw2mXi1Iukl9Swex3Etiedor/uXUQv4y3rwuTngDGBt1qrtyolGtpoCvlArlF5dz15hYOnCzgjduHcv3AzmcPpm0B8bHmP1eqEWmgK+UCT3++i4MnC5l/Vzzj+nY492DaJuh4CQS1cU9xymtoH7pSF+n7A1l8mXSc+6/udX6YV1VCeqL2n6sm4dQduohMAF4FfIEFxpgXaxz/OzDOvhkMdDDGtHNloUo1R6UVVTzz+S5iI0L49VU9rJ2Z+2DVY9aaoZVl1nS52n+umkC9gS4ivsBc4FogHUgQkeX2ZecAMMY8VK3974EhjVCrUs3OvHWHSM0p5t3Zwwn087V2bnwNjm6x+sz9W0HvidDrGvcWqryCM3fow4BkY0wKgIgsA6YCe2ppPwP4s2vKU6r5SskqZN66Q0we3IUr4iKsnaWnYfenMGgaTHndvQUqr+NMH3pX4Gi17XT7vvOISHcgFlhz8aUp1XztPX6aOxZsIdDfh6cn9Tt7YPcnUFEMQ2a6rzjltZwJdEcz75ta2k4HPjLGVDl8I5E5IpIoIolZWVnO1qhUs7Jufya3ztuIMbBszgg6tAk6e3D7YojsZz0VqlQTcybQ04Fu1bajgIxa2k4H3qvtjYwx840x8caY+MhIXdVctTyfbE9n9sJEuoeH8OnvRnFJl7ZnD2buhWOJMPQuEF2BSDU9ZwI9AYgTkVgRCcAK7eU1G4lIH6A9sMm1JSrVPGxJyeHxj5MYFhPGB78ZSee2rc5tsH0x+PjDoOnuKVB5vXoD3RhTCdwPfA3sBT4wxuwWkedFZEq1pjOAZcaY2rpjlGqxjuYW85t3t9EtLJh/3nUZoYE1xhNUlsGP70HfSRAS7p4ilddzahy6MWYlsLLGvmdqbD/rurKUaj4KSiuYvTABm4E3Z11O21YOZkzcuwJKcq3uFqXcRJ8UVaoOKVmF3LlgC4eyiph7+1BiHa02VFEKq5+3vgztMe7840o1EZ3LRSkHbDbDu1uO8N8r9xLo58vc24ecHWte06Y3IO8IzPwcfHybtlClqtFAV6oGm83w+2U7+DLpOFf1juTlWwfRsfrQxOpOZ8D6v0HfG6DH2KYsU6nzaKArVcNf/72fL5OO8+h1ffiPsT2RuoYgfvusNWfLdX9psvqUqo32oStVzWc7jjF37SFmDOtWf5in/gBJ78Oo30P7mCarUanaaKArZbcj7RSPfZzE8NgwnpsyoO4w378KlkyzgnzMH5usRqXqooGuFLDpUA53v51ApzZBzLvzMgL86virsfmfsOx2iOwN93wDAQ5GvijlBhroyut9vC2dmW9tIbJ1IEvuHU5YSEDtjX/4B3z1OPS5Hu7+Elp3bLpClaqHfimqvNobaw7yyjcHGNUznHl3Xub4oaEz8tJg3QvWiJZpi3SIomp2NNCV1/r+QBavfHOAGy/twsu3Dq67mwXgm6cAgYkvaZirZkm7XJRXyi+u4LGPkojrEMqLtwyqP8wPfw97Pre+AG0b1TRFKtVAeoeuvNKzK3aTXVjGv2bGE+Rfz912VSWsegLaRVtDFJVqpjTQlddZ9dNxPt1xjAfHxzEwqm39L9gyDzJ3W/3m/q3qb6+Um2igK6+yZt9JHvs4iUFRbfnduF51NzYGvnsZ1v039J4A/abU3V4pN9NAV16hymb4x7cHeH1NMv07t+F/7xiKv28d/eaV5bDiD9Yc54NnwOTXdBUi1expoCuPl1NYxgPLdvJDcjbT4qN4fuqAuvvNi3Phg5mQuh7GPQlXPqphrloEDXTl0banneJ3S7aTU1TOS7cM5LbLo+t+QW6K9Uh/3hG4+V8waFrTFKqUCzg1bFFEJojIfhFJFpEnamkzTUT2iMhuEVnq2jKVahhjDIs3pXLb/23Cz1f45Lej6g/ztC2wYDwUZ8Ndn2mYqxan3jt0EfEF5gLXAulAgogsN8bsqdYmDvgTMNoYc0pEOjRWwUrVp7LKxrMrdvPu5jSu7tuBv0+7lLbBdTwBCrDrY/j0t9C2K9z+IUTU84WpUs2QM10uw4BkY0wKgIgsA6YCe6q1uQ+Ya4w5BWCMyXR1oUo5o6C0gvuX7uC7A1n85qqePHZdH3x86uj/NgZ++Ju1hFy3ETB9qS7yrFosZwK9K3C02nY6MLxGm94AIrIB8AWeNcZ85ZIKlXLSydOlzHprKwczC3nh5oHMGFZPF0tVBXzxIOx4FwbcClPngn8tKxMp1QI4E+iObm+Mg/eJA8YCUcB6ERlgjMk7541E5gBzAKKj6/nLplQDHM0t5o4FW8gpLOPtuy/nyt6Rdb+gJM8ayXL4O7jyMRj3/3Qki2rxnAn0dKBbte0oIMNBm83GmArgsIjsxwr4hOqNjDHzgfkA8fHxNf+noNQFSc4s5M4FWyipqOLde4czJLp93S84lWqNZMlNgRvnwaW3N0mdSjU2Z0a5JABxIhIrIgHAdGB5jTafAeMARCQCqwsmxZWFKuXIhuRsbvu/TVTaDMvmjKg/zNMTrZEshSfgrk80zJVHqfcO3RhTKSL3A19j9Y+/ZYzZLSLPA4nGmOX2Y78QkT1AFfCoMSanMQtX3q2kvIqXvtrHOxtT6REZwoKZ8fSIDK37RXs+h0/mQGhHuHulteKQUh5EjHFPz0d8fLxJTEx0y2erliuvuJyvdp1g/voUUrKKuHtUDI9P6EurgDqe/DQGNr4O/34GouJh+nsQWk8fu1LNlIhsM8bEOzqmT4qqFiEpPY9/fHuQ7w9kUWkz9IwMYcm9wxndK6LuF1ZVwMpHYNs7cMlNVp+5zpioPJQGumrWKqpsvLEmmTfWJhMWEsDsK2KZPLgLl3Rpg9Q3KqU0Hz68Gw6tgSv+CFc/DT66povyXBroqtlKzizgofd/5Kdj+dw8pCt/nnJJ3Wt+Vpd3FJZOg+wDMOUNGHpX4xarVDOgga6aHZvN8PbGVF76ah8hAb7Mu2MoEwd2dv4NMnbA0tugohTu+Ah6jmu8YpVqRjTQVbNyLK+Ehz/YyeaUXK7p24EXbhlIh9YNeHpz35fw8b0QHAEzl0OHvo1XrFLNjAa6aja+O5DFA8t2UFFp46VbBjItvlv9/eTVndwN798JXYbAjGUQqnPEKe+iga7czmYzvLbmIK+uPkifjq2Zd+dlxEaENPyNtr0DPv5WN0twmMvrVKq500BXblNaUcUXScd5e8Nhdmec5uahXfnLjQPrHlNem4oSSHof+k3WMFdeSwNdNbmC0gre3pDKwo2p5BSVE9chlL/fNpgbL+3asC6W6vZ+YQ1THDrTtcUq1YJooKsmU1JexcJNqfzzu0PkFVdwdd8O3DM6ltG9wi88yM/YsQjadYeYMS6pVamWSANdNbqMvBIWbz7Ce1vTyCuu4KrekTz8i94Mimrnmg/ITYHD38O4p/TBIeXVNNBVoymvtPHn5bv4IDEdYwzX9u/IfWN6EB/j4j7uHUtAfHTmROX1NNBVoygpr+K3S7axbn8Wd4+K4d4xsUS1D3b9B9mqYOdS6DXeWg9UKS+mga5crqC0gtkLE0lIzXVuKbgLVVUJXz0BBRlw/cuN8xlKtSAa6MpljuWV8GVSBssSjpKWU8yr04cwZXCXxvmwskL46B44+DWMvB/6TGqcz1GqBdFAVxelosrG17tPsGjTEbYezgVgcFRbFsyKZ2wfFz6pWZgJW/8FlaXW9qG1kLkHJv0NLp/tus9RqgXTQFcXpKS8irc3HmbRxiOcOF1K9/BgHvlFbyYP7kL38At4yrM+y/8AB74CP/u8LkFt4fYPIG686z9LqRZKA101iDGGb/dm8tyK3aSfKmFMXAT/ffMAxvbugI/PRY4lr03yt3BgFYx/Dq54sHE+QykP4FSgi8gE4FWsNUUXGGNerHH8buB/gGP2XW8YYxa4sE7lZsYYth05xdy1yazdn0XvjqEsmzOCET3CG/eDqyrgqz9BWA8Y8dvG/SylWrh6A11EfIG5wLVAOpAgIsuNMXtqNH3fGHN/I9So3CinsIzV+zJZtCmVXcdO0ybIj6cm9WPWqBj8fZvgIZ6t861FKma8D36Bjf95SrVgztyhDwOSjTEpACKyDJgK1Ax05SHSTxXz9oZUNiRns+9EAQBxHUL5y00DuGlIV4IDXNxTV5oPx388f39VOax70Rpj3vs6136mUh7Imb+ZXYGj1bbTgeEO2t0iIlcCB4CHjDFHazYQkTnAHIDo6EYam6wuWFllFQvWH+b1NQexGRgWE8aj13VhVM9wLu3W7uLnW3Ekcy8s+SXkn/fHxeIbANe9AI3x2Up5GGcC3dHfJFNjewXwnjGmTER+AywErj7vRcbMB+YDxMfH13wP1QQOZxfx1a4THDhZwP4TBWQWlNIuOIDwkABOnC7lSE4xEy7pxNOT+9O1XavGLebQWvhgJvi3gtvehVbtz2/TLtr6UUrVy5lATwe6VduOAjKqNzDG5FTb/Bfw0sWXphriVFE5HyQe5VBW4c/7wkMDGdkjnPiY9uQUlvP6moN8vP0YVTZDl7ZB9O7UmsHd2pFfUk52QTkdWwfx3JRLLm78eFUlbHodcpLrbldZDrs/gYje1vDDdt3qbq+UqpczgZ4AxIlILNYolunAObMgiUhnY8xx++YUYK9Lq1S12nv8NAs3pvLpjmOUVdro2CYQHxGMgezCMuatO4S/r/WPLEGYObI7v72qJx3aNGCdTmeVFdif3vwGWne2JsyqS99JMOUNCGrj+lqU8kL1BroxplJE7ge+xhq2+JYxZreIPA8kGmOWA38QkSlAJZAL3N2INXuV4vJKcgrLySkqJ8jfh/CQQNq08mPtvize2XiYzSm5BPn7cPPQKO4eFUOfTq3PeW1C6ik2JmdjM4ZfjY6lS2N1o+Qfg6W36dObSrmRGOOeruz4+HiTmJjols9u7mw2w8fb03nlm/2cPF1Wa7uu7Voxc2R3bru8G+2CA5qwwhqO/2iFeVkhTHvHGpWilGoUIrLNGBPv6Jg+KepGVTbDsoQ03t6QSvewYEb2DKdnh1Dmrkkm8cgphkS341ejYwkLCSAsOIDyKhs5hWXkFJXTt1MbxvfrgF9TjAWvy/6vrG6WVu1h9tfQ8RL31qOUF9NAd5OdR/N45vNdJKXnMyiqLSnZRazelwlAWEgAL986iFuHRjXe4/QXwhg4/B2U5Fnb2Qdg3QvQaRDc/j607uTe+pTychroTay0oopXvt7PmxsOExkayKvTL2XK4C6ICBl5Jew6ls+w2DD3dqE4UlkOKx6AH5eeu7/P9XDLAghohAm5lFINooHehHYdy+eh93dyMLOQO0dE8/iEvrQO8v/5eJd2rRrvS8uLUXIK3r8LUtfDVY9D/xut/b7+EN5LH/pRqpnQQG8CJ0+XMndtMku3pBEeGsDCe4ZxVe9Id5dVu5N7YP1fz849fuInKDgON82Hwbe5tzalVK000BtRfkkFb6w5yKJNR6iyGX4Z340nJvSlbbB//S92l+TV8OHd1l13W/vDPqEd4MZ5EDParaUppeqmgd5I1h/M4tEPk8gsKOWmIVE8cE0c0eGNsEiyM84MTXXUNVJRAsZm/f6nD+GLP0JkX7jjA2gb1XQ1KqUumga6i5WUV/HCqr0s2nSEXh1CmT9zNIOi2rm3qHUvWGF927tnhxVWVcKqxyDxzXPb9hoPt76tT28q1QJpoLvQ9rRTPPzBj6TmFDH7ilgeva4PQf6+7i2qogS2/NOaovbN66wHf6KGWd0qh1bD0FkQ3tNq2yoMBs8AX/1joVRLpH9zXaC80sZrqw/yv+uS6dy2FUvuHc6onhHuLsuy9wsrzG/8J2yeC0umWRNh5adb86gMvcvdFSqlXEQD3QWeW7GbJVvS+OVlUTwzuf85QxHdbsciaNcdBt0G/SbDx7MhbRPc8RH0HOfu6pRSLqSBfpFSsgpZlnCUWSO789zUAe4u51y5h+Hw9zDuKfDxgcBQmLHMWglIl3NTyuO4eSKQlu+v/z5AoJ8Pv78mzt2lnG/nEmsK20urzXYsomGulIfSQL8Iu47l82XSce69IpaI0GYWkrYq2LEEel4Dbbu6uxqlVBPQLpeL8PLX+2kf7M+9V/ZwXxEVpVCcff7+tM1QkAETX2z6mpRSbqGBfoE2Hcrh+wNZPHl9P9q460tQY+Ct6+D4TsfHgyOg98SmrUkp5TYa6BegoLSCJz/7ic5tg7hrZHf3FXJsmxXm8bOhy6XnH+80CPya2ayNSqlG41Sgi8gE4FWsJegWGGMc/jteRG4FPgQuN8Z45HJExhge+yiJIznFLL13uHsfHNq+CPyDYfyz+mSnUqr+L0VFxBeYC0wE+gMzRKS/g3atgT8AW1xdZHPy5g+HWbXrBE9M6MvwHuHuK6SsEHZ9DJfcpGGulAKcG+UyDEg2xqQYY8qBZcBUB+3+E3gZKHVhfc3K1sO5vLBqHxMHdOLeMbHuLWbPZ1BeCEP0SU+llMWZLpeuwNFq2+nA8OoNRGQI0M0Y84WIPOLC+pqNiiobj3+cRHRYMC/fOghx1aIOhZmw7A4oK7C2ff1hzMNwyY11v277YgiPg+gRrqlDKdXiOXOH7ii5zM8HRXyAvwMP1/tGInNEJFFEErOyspyvshn4MDGdw9lFPDWpn2sf7d+xGNK3WhNkRcSBrRI+nAU//P3stLc1ZR2Ao5thyJ26WpBS6mfO3KGnA92qbUcBGdW2WwMDgHX2u9ZOwHIRmVLzi1FjzHxgPkB8fHwtadX8lFZU8erqA1zWvT1X9+3g/AttNijNg+Cw2o9vXwzdr4DpS6x9FaXw2W/h22chNwUuv+/81yW+CT5+1syISill50ygJwBxIhILHAOmAz8/S26MyQd+nlpQRNYBj3jSKJeFG1M5ebqM12cMdb6rpawQProHUtbCfWuhk4N5Xo5sgFOHYewTZ/f5B8Etb0JYD1j/ijWSxZG+N0Drjg0/GaWUx6o30I0xlSJyP/A11rDFt4wxu0XkeSDRGLO8sYt0p9OlFcz77hBj+0QyLLaWO+3zXpQBS6fByd3WsMKvnoBZK87vHtmxGALbQr8p5+738YFrnoY+11treToSPbLhJ6OU8iGgkYsAAA9uSURBVGhOjUM3xqwEVtbY90wtbcdefFnNx/zvUsgrruCRX/SpvdGxbbD7M8BY/d67PoGy03D7B5B3BL58GPZ8fu4XnSV51r5L74CAWpami7rMpeeilPJs+qRoHZIzC5m/PoWpl3ZhQNe2jhv99JHV522MNUIFrLU47/gAOg20JslKfBu+eRp6Xwf+reyv+xAqS3WBCaWUy2ig18JmM/zpkyRa+fvy1KTznqOyAnz9K7Dmv6zuj+lLHX/56eMLE1+CdybBhtdg7OPW/h2LoeNA6OzgkX2llLoAGui1WLLlCAmpp3jll4OJbO1gatzv/wfW/gUGToOpb9Q9x3jMFdD/RvjuJWuEijFQlAkTX9Zhh0opl9FAd+BYXgkvrtrHmLgIbhnqYC7xyjLY/L/WTIY3z3culK9/xeqKKS+0tv1DrP5zpZRyEQ10B/78+W4M8N83DXQ8THHfF1ByCobd5/wddmgkXPcXl9aplFLV6YpFNSRnFvLt3pP85qqedAurZfTJ9sXQthv00EWWlVLNhwZ6De9uPoK/r3D78GjHDfLSIGWd1V3io//5lFLNhyZSNcXllXy8LZ3rB3aufY3QHfZH9Ido/7dSqnnRQK/msx0ZFJRVcteIWlYhslXBziXQcxy0q+UOXiml3EQD3c4Yw6JNqfTt1JrLurd33ChlHeQf1TnIlVLNkga63fa0U+w7UcDMkTG1T8CVsABahUHfSU1bnFJKOUED3W7RpiO0DvRj6qVdHDc4/D3sXwkj/qPuh4iUUspNNNCB3KJyVv10gpuHdiUk0MHQ/KpKWPW41W8+6v6mL1AppZygDxYBn+44RnmVjenDavmic9vbkLkHpi0+O7mWUko1M15/h26M4YOEowyOaku/zm3Ob1Cca03AFXsl9Jvc9AUqpZSTvD7Qf0zPZ//JAqZd3s1xg7V/sRZwnvCSTqSllGrWvD7Q3084SpC/D5MHO/gy9MQuSHwLLp8NHR1MoauUUs2IVwd6cXklK37MYNLALrQJ8j/3oDHW0nFB7WDsn9xToFJKNYBTgS4iE0Rkv4gki8gTDo7/RkR+EpGdIvKDiLSI29mVP52gsKyS2xx1t+xdDqnr4eonHS9coZRSzUy9gS4ivsBcYCLQH5jhILCXGmMGGmMuBV4G/ubyShvB+wlp9IgI4fKYGk+GVpTA109Bh0tg6N1uqU0ppRrKmTv0YUCyMSbFGFMOLAOmVm9gjDldbTMEMK4rsXEczy8hIfUUt1wWdf6ToRteg/w0a+k4Xx3ZqZRqGZxJq67A0Wrb6cDwmo1E5HfAH4EA4GpHbyQic4A5ANHR7p3cavXeTACuu6Tj2Z3GwKY3YN0LcMlNEDvGTdUppVTDOXOH7mis3nl34MaYucaYnsDjwFOO3sgYM98YE2+MiY+MjGxYpS62Zl8m0WHB9IwMtXZUVcIXD8E3T0H/qXDjPLfWp5RSDeVMoKcD1b81jAIy6mi/DLjxYopqbCXlVWxIzuaafh2s7pbS07B0mvVE6BUPwa1v6xOhSqkWx5kulwQgTkRigWPAdOD26g1EJM4Yc9C+OQk4SDO2ITmbskob1/TtCPnpsGQaZO2Dya/BZbPcXZ5SSl2QegPdGFMpIvcDXwO+wFvGmN0i8jyQaIxZDtwvIuOBCuAU0KxTcfW+TEID/RgelAb/mgEVxXDnR9DTYde/Ukq1CE4N4TDGrARW1tj3TLXfP+DiuhqNMYY1+05yda82+C+5CQLbwMzPoEM/d5emlFIXxeueFN2dcZqTp8u4qWMmlObBxBc1zJVSHsHrAv3bvScRgWG+B6wd3Ua4tyCllHIRrwv0NfsyGdKtHSEnEiCiN4SEu7skpZRyCa8K9MPZRSSl53NN30g4ugW6nfd8lFJKtVheFeivrT5IkL8PM2JLrP7z6JHuLkkppVzGawI9ObOQz3ceY+bIGMJytls7o7X/XCnlObwm0F9bfZBAP1/mXNkD0jZDSCSE9XB3WUop5TJeEegHTxawIimDWaNiiAgNhKObrbtzXVJOKeVBvCLQ/7H6IMH+9rvz08fhVKoOV1RKeRyPD/TU7CJW/nScWaNiCAsJsO7OQb8QVUp5HI8P9BU/ZmAM3DWyu7UjbQv4tYLOg9xbmFJKuZjnB3pSBsNiwujc1j4dbtomiIoHX/+6X6iUUi2MRwf6/hMFHDhZyA2DO1s7MnbCiZ90uKJSyiN5dKB/kZSBj8DEAZ1h/1fw9vXQpgsMucvdpSmllMt5bKAbY1jxYwYje4YTuWchLJsBEXFw77fQvru7y1NKKZfz2EDfdew0qTnF3N6jHFY9CnHXwa9WQutO7i5NKaUahccG+hdJGfj5CGPbHrd2XP0kBIS4tyillGpETgW6iEwQkf0ikiwiTzg4/kcR2SMiSSKyWkTc2qdhsxm+SDrOmLgIQvKTQXwgPM6dJSmlVKOrN9BFxBeYC0wE+gMzRKR/jWY7gHhjzCDgI+BlVxfaEJsP53Asr4QbBnWxFn8O6wH+Qe4sSSmlGp0zd+jDgGRjTIoxphxYBkyt3sAYs9YYU2zf3AxEubbMhpm37hARoQFMGtTZCvTIvu4sRymlmoQzgd4VOFptO92+rzazgVUXU9TF2JF2ivUHs7lvTA+CpApyDmmgK6W8gp8TbRxNSWgcNhS5E4gHrqrl+BxgDkB0dLSTJTbM3LXJtAv2544R3SHnAJgqDXSllFdw5g49HehWbTsKyKjZSETGA08CU4wxZY7eyBgz3xgTb4yJj4yMvJB667Q7I59v92Zyz+hYQgP9IGuvdaCDBrpSyvM5E+gJQJyIxIpIADAdWF69gYgMAf4PK8wzXV+mc/537SFaB/oxa1SMtSNrv45wUUp5jXoD3RhTCdwPfA3sBT4wxuwWkedFZIq92f8AocCHIrJTRJbX8naNJjmzgJW7jjNzVHfatrJPvJW5F9rH6ggXpZRXcKYPHWPMSmBljX3PVPv9eBfX1WD//C6FQD8f7hkde3Zn1n7tP1dKeQ2PeFL0eH4Jn+88xm3x3QgPDbR2VpZD7iHtP1dKeQ2PCPQ31x/GZuDeMdUWfc49BLZKvUNXSnmNFh/o+cUVvLc1jcmDOtMtLPjsgUz7CBcNdKWUl2jxgb54cypF5VX8+qqe5x7I2meNcInQES5KKe/QogO9tKKKtzekMrZPJP06tzn3YNY+aB8D/q3cUptSSjW1Fh3oy7amkVNUzm9q3p0DZOocLkop79JiA72orJI31iYzokcYw2PDzj14ZoSLBrpSyos4NQ69OXrrh8NkF5Yzf2ZfRATStsC6F6CyFCrLdISLUsrrtMg79FNF5cz/PoVr+3dkaHR72PUxLJwM2QfA1x8CQ6H3ROg5zt2lKqVUk2mRd+j//O4QheWVPHJtb/j+FVjznxA9EqYvheCw+t9AKaU8UIsL9BP5pbyzMZWbhnSlz9H3rTAfOA2mvgF+ge4uTyml3KbFdbks2XIEmzE8PDrcCvMeY+Hm+RrmSimv1+Lu0B+4Jo5xfTvQdcfzUFYIE14CcbQGh1JKeZcWd4fu5+vD0IB02PYODLtPJ99SSim7FhfoGAOrHoegdjD2CXdXo5RSzUaL63Jh96dwZAPc8Hdo1d7d1SilVLPR8u7QA1tDn0kwdJa7K1FKqWal5d2hx11r/SillDqHU3foIjJBRPaLSLKInNdxLSJXish2EakUkVtdX6ZSSqn61BvoIuILzAUmAv2BGSLSv0azNOBuYKmrC1RKKeUcZ7pchgHJxpgUABFZBkwF9pxpYIxJtR+zNUKNSimlnOBMl0tX4Gi17XT7vgYTkTkikigiiVlZWRfyFkoppWrhTKA7egzTXMiHGWPmG2PijTHxkZGRF/IWSimlauFMoKcD3aptRwEZjVOOUkqpC+VMoCcAcSISKyIBwHRgeeOWpZRSqqHqDXRjTCVwP/A1sBf4wBizW0SeF5EpACJyuYikA78E/k9Edjdm0Uoppc4nxlxQd/jFf7BIFnDkAl8eAWS7sJyWwhvP2xvPGbzzvL3xnKHh593dGOPwS0i3BfrFEJFEY0y8u+toat543t54zuCd5+2N5wyuPe+WN5eLUkophzTQlVLKQ7TUQJ/v7gLcxBvP2xvPGbzzvL3xnMGF590i+9CVUkqdr6XeoSullKqhxQV6fVP5egIR6SYia0Vkr4jsFpEH7PvDROTfInLQ/qvHLdkkIr4iskNEvrBvx4rIFvs5v29/uM2jiEg7EflIRPbZr/lIL7nWD9n/fO8SkfdEJMjTrreIvCUimSKyq9o+h9dWLK/Zsy1JRIY29PNaVKA7OZWvJ6gEHjbG9ANGAL+zn+cTwGpjTByw2r7taR7AeoDtjJeAv9vP+RQw2y1VNa5Xga+MMX2BwVjn79HXWkS6An8A4o0xAwBfrKfQPe16vwNMqLGvtms7EYiz/8wB5jX0w1pUoFNtKl9jTDlwZipfj2KMOW6M2W7/fQHWX/CuWOe60N5sIXCjeypsHCISBUwCFti3Bbga+MjexBPPuQ1wJfAmgDGm3BiTh4dfazs/oJWI+AHBwHE87HobY74Hcmvsru3aTgUWGctmoJ2IdG7I57W0QHfZVL4thYjEAEOALUBHY8xxsEIf6OC+yhrFP4DHgDPz6ocDefbpJ8Azr3cPIAt4297VtEBEQvDwa22MOQa8grU4znEgH9iG519vqP3aXnS+tbRAd9lUvi2BiIQCHwMPGmNOu7uexiQiNwCZxpht1Xc7aOpp19sPGArMM8YMAYrwsO4VR+z9xlOBWKALEILV5VCTp13vulz0n/eWFuheM5WviPhjhfkSY8wn9t0nz/wTzP5rprvqawSjgSkikorVlXY11h17O/s/ycEzr3c6kG6M2WLf/ggr4D35WgOMBw4bY7KMMRXAJ8AoPP96Q+3X9qLzraUFuldM5WvvO34T2GuM+Vu1Q8uBWfbfzwI+b+raGosx5k/GmChjTAzWdV1jjLkDWAucWXjco84ZwBhzAjgqIn3su67BWt7RY6+1XRowQkSC7X/ez5y3R19vu9qu7XJgpn20ywgg/0zXjNOMMS3qB7geOAAcAp50dz2NdI5XYP1TKwnYaf+5HqtPeTVw0P5rmLtrbaTzHwt8Yf99D2ArkAx8CAS6u75GON9LgUT79f4MaO8N1xp4DtgH7AIWA4Gedr2B97C+I6jAugOfXdu1xepymWvPtp+wRgA16PP0SVGllPIQLa3LRSmlVC000JVSykNooCullIfQQFdKKQ+hga6UUh5CA10ppTyEBrpSSnkIDXSllPIQ/x8kCgIBA9W69QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(train_scores)\n",
    "plt.plot(test_scores)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ReLU + Sigmoid + Cross Entropy + L2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "import torch.optim as optim\n",
    "net2 = Net()\n",
    "criterion2 = nn.CrossEntropyLoss()\n",
    "optimizer2 = optim.SGD(net2.parameters(), lr = 1e-1, weight_decay=1e-2)\n",
    "loss_scores2, train_scores2, test_scores2 = fit(net2, criterion2, optimizer2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXhV1b3/8fc3M0MIGSEQIIQZlckDCYOK89hinXEAEUTqWFtbewd7297b++utVeuEyiSIirN1qHNRUSFAIvMgEgIYpoQwhSkhZN0/cvTHxSQEcsLOOefzep485Jy9svd3u+Hjzjprr2XOOUREJPhFeF2AiIgEhgJdRCREKNBFREKEAl1EJEQo0EVEQkSUVwdOSUlxmZmZXh1eRCQo5efnb3fOpda0zbNAz8zMJC8vz6vDi4gEJTPbUNs2dbmIiIQIBbqISIhQoIuIhAgFuohIiFCgi4iECAW6iEiIUKCLiISIoAv09dv38dBH3zC3YDsHDx32uhwRkSbDsweLTtTSTbuZ+FkBj89eS0xUBKd3TGRwl2QGd0mmb0ZrYqKC7v9RIiIBYV4tcOHz+dyJPiladvAQC9fvYO7aUuYWlLJq6x6cg2bRkZzeKZHszknkdEmmT0YCsVGRAa5cRMQ7ZpbvnPPVuC0YA/1ou/ZXkLtuB7nrSsldV8rqrWUAxEZFMKBjIoM6J5GdlUT/Dok0i1HAi0jwCvlAP9rOfRUsWL+D+et2ML+wlJVbqu/goyONvhmtGdQ5iYGdk/B1SiQ+LrpRahARaQxhF+hH233gEF9v2Mn8wuqAX1a0m8oqR4RB73atGJiZRHbnJAZmJpHcMvak1CQiciLCPtCPtr+iksUbd5FbuIOFhTv4euNOyiurAOia1rK6i6ZzEoM6J5Ge0MyTGkVEaqJAP4aKyiqWbdrFgsKdLCgsJW/9TsrKKwHokNSMQZnJPwR8p+TmmJnHFYtIuFKgH6fDVY5VW/awwN9Fs3D9TnbsqwCgTatYBnWuDvicrGS6pLZQwIvISaNAbyDnHAUle8ldt+OHkN+2pxyAlJaxZPtH0eRkJdMtraUCXkQaTV2BHnQPFnnBzOiaFk/XtHhuzOmEc44NpfuZX1jKfP9wyX8s2wJAcosYsrOSGJyVTE5WMl0V8CJykijQT4CZkZnSgsyUFlw7sCPOOb7bcYDcwupx8LkFpby3bCtQfQefk5XEkC4pDOmSrD54EWk0xwx0M+sAPAe0BaqASc65R49qcwNwv//lXuDnzrklAa61yTIzOiY3p2Nyc67xdfgh4Oet2868glLmrSvl3aXVd/DtEuIY0rU63Id2TaFNqziPqxeRUHHMPnQzSwfSnXNfm1k8kA9c7pxbeUSbIcAq59xOM7sY+L1zLruu/QZTH3pDOedYt30fcwtKmVdQHfI79x8CqodJDuuawtCuKeRkJelBJxGpU0A/FDWzt4AnnHMf17I9EVjunGtf137CKdCPVlXlWLllD3MLtvPl2lIWFJZy8FAVkRFGvw6tOaNbCmd0S6VvRgJRkZpsTET+v4AFupllAnOAU51ze2ppcx/Q0zk3roZt44HxAB07djx9w4YN9T52KCuvPMzXG3bx5doSvvx2O0s37cY5iI+LYkiXZM7qnsaZ3VPISGzudaki4rGABLqZtQQ+B/7knHujljZnAxOBYc650rr2F8536Meyc18FcwtK+eLbEuasKWHz7oMAdEltwfAeaQzvkcqgzkmaSVIkDDU40M0sGngX+NA593AtbfoAbwIXO+fWHGufCvT6+X4M/GfflPD5mhLmF+6gorKK5jGRDOmSwtk9UzmnZ5qmKBAJEw0KdKseYzcD2OGc+0UtbToCs4FRzrm59SlKgX5i9ldUkruulE9XlzB7dTGbdh0AoHd6K87tlca5vdrQp30CEREaGikSihoa6MOAL4BlVA9bBPhXoCOAc+5pM5sCXAl83yleWdsBv6dAbzjnHGuL9zJ7dTH/XF1M/oadHK5ypMbHcm7PNM7r1YZh3VKIi1bXjEio0KP/YWLX/go++6aEj1dt4/NvSthbXknzmEjO6p7KBae04ZyebUhopmGRIsFMgR6GKiqryF1Xykcrt/LRim0Ul5UTHWkM6ZLCRae25cJT2pLUIsbrMkXkOCnQw1xVlWNx0S4+XL6V95dvZeOO/URGGIOzkrn4tLZcfGq6wl0kSCjQ5QfOVT/U9N6yLby3bCuF2/cRGWEM7ZrCT/qkc+GpbWmlp1VFmiwFutTo+3B/d+kW3l26me92HCAmKoJzeqQxol87zu6Zpg9URZoYBbock3OOxd/t4u0lm3lnyRa27y0nPi6KS09L5/L+7RmUmaShkCJNgAJdjsvhKsfcgu28uWgTHyzfyv6Kw2QkNuPKARlcdXoGHZI0BYGIVxTocsL2V1Ty4YqtvJ6/ia8KtuMc5GQlce3ADlx8arq6ZEROMgW6BMSmXQd4I7+IV/OL2LhjP/FxUfysf3uuz+5Iz7atvC5PJCwo0CWgqqocuYWlvLLwO95bvpWKyir6d2zNDdmduKyP7tpFGpMCXRrNzn0VvLFoEy/O30BByT5aN4/mGl8HbszuRMdk9bWLBJoCXRqdc47cdTt4PncDH67YymHnOLdnG24eksnQrslaR1UkQOoKdC0SLQFhZgzukszgLsls23OQF3I38ML8jXyyahvd27Rk7LDOjOjXXt0xIo1Id+jSaA4eOsy7S7cw9ctCVm3ZQ0rLGG7KyWTU4E4kaqoBkROiLhfxlHOOeetKmfJFIbNXF9MsOpJrB3Zg7LDOGtMucpwU6NJkfLutjGfmrOOtxZuocnB5v/b8fHgXuqa19Lo0kaCgQJcmZ8vuA0yeU8iLCzZQXlnFJaemc9e5XTWeXeQYFOjSZJXuLWfaV4XMmLuBveWVXHxqW+4+txu90hXsIjVRoEuTt2t/BdO+LOTZr9ZTVl7JZX3Suff87nRJVVeMyJHqCvSIevxwBzP71MxWmdkKM7unhjZmZo+Z2VozW2pmAwJRuISP1s1j+OUFPfjy/nO48+yuzF5dzPkPf86vX13yw0LYIlK3+iwSnQ6kO+e+NrN4IB+43Dm38og2lwB3AZcA2cCjzrnsuvarO3Spy/a95Uz8tIDnczeAwZghmdw+vCsJzbX4hoS3Bt2hO+e2OOe+9n9fBqwC2h/VbATwnKuWC7T2/49A5ISktIzldz/pzez7zuKyPulM+mIdZ/xlNlO+WEdFZZXX5Yk0SccM9COZWSbQH5h/1Kb2wHdHvC7ix6GPmY03szwzyyspKTm+SiUsZSQ25+Fr+vHe3WfQt0Nr/usfq7jgkc/5cMVWvPr8R6Spqnegm1lL4HXgF865PUdvruFHfvSvzTk3yTnnc875UlNTj69SCWu90lsxc2w2z44ZSFRkBLfNzOfGqfP5dluZ16WJNBn1CnQzi6Y6zF9wzr1RQ5MioMMRrzOAzQ0vT+T/OrtHGh/ccwZ/+OkpLCvazcWPfsF/vruSsoOHvC5NxHP1GeViwFRglXPu4VqavQ2M8o92yQF2O+e2BLBOkR9ERUYwekgmn943nKt9GUz7qpBzH/qcfyzdom4YCWv1GeUyDPgCWAZ8/2nUvwIdAZxzT/tD/wngImA/MMY5V+cQFo1ykUBZ8t0u/vXNZazYvIczu6fyXyNO1VzsErL0YJGEvMrDVczM3cBDH62hsqqKX53fgzFDM4mKPK7P/UWavAYNWxQJBlGREYwZ2pmPf3kmw7qm8Kf3VnHFU3NZteXoz+9FQpcCXUJKekIzJo/y8fjI/mzaeYCfPvElT8z+lsrDGrsuoU+BLiHHzPhJ33Z8/MuzuOCUtvz1ozVc+dRc1hZriKOENgW6hKykFjE8ef0Anri+Pxt37OfSx75k5rz1GgkjIUuBLiHvsj7t+PDeM8nJSuaBt1Zwy/SFlJSVe12WSMAp0CUspMXHMX3MQP7w01OYW1DKxY9+wRffavoJCS0KdAkbZsboIZm8c9cwEptHM2raAv7ywWp9YCohQ4EuYad7m3jevnMY1/o6MPGzAkZOzmXbnoNelyXSYAp0CUvNYiL585V9ePS6fizftIdLH/uS3HWlXpcl0iAKdAlrI/q15607h9KqWRQ3TJnPM58XaBSMBC0FuoS977tgLjqlLf/v/dXc89JiDlQc9roskeOmQBcBWsZG8cT1/fn1hT14Z+lmrnp6rtYylaCjQBfxMzPuOLsrU0f72Fi6nxFPfEn+hp1elyVSbwp0kaOc07MNb94xlBaxUYycnMvbS7RWiwQHBbpIDbqmteTN24fSL6M1d89axKOffKsPS6XJU6CL1CKpRQwzxw3iiv7teeSTNdz/+lIO6SEkacKivC5ApCmLjYrkoWv6kpHYjMdmr6W4rJwnrx9Ai1j905GmR3foIsdgZvzygh786WenMmdNCSMn51K6V5N7SdNTn0Wip5lZsZktr2V7gpm9Y2ZLzGyFmY0JfJki3rshuxOTbvLxzdYyrn5mnoY1SpNTnzv06VQv/lybO4CVzrm+wHDgITOLaXhpIk3Peb3bMHNsNiV7yrnqqbmsLd7rdUkiPzhmoDvn5gA76moCxJuZAS39bSsDU55I0zOocxIv3ZbDocOOa56Zx/JNu70uSQQITB/6E0AvYDOwDLjHOVfjUAAzG29meWaWV1KiuagleJ3SLoHXJgymWXQk10/OZdFGPYAk3gtEoF8ILAbaAf2AJ8ysVU0NnXOTnHM+55wvNTU1AIcW8U5mSgtevi2HxBYx3DR1AQvX1/WLrEjjC0SgjwHecNXWAoVAzwDsV6TJy0hszsvjB9OmVSyjpi5gXoGm4BXvBCLQNwLnAphZG6AHsC4A+xUJCm0T4nhp/GA6JDXjlukLNa+6eKY+wxZnAfOAHmZWZGZjzWyCmU3wN/lPYIiZLQP+CdzvnNveeCWLND2p8bG8eGsOGYnNGPOsQl28YV7NT+Hz+VxeXp4nxxZpLCVl5YycnMumnQeYccsgBnVO8rokCTFmlu+c89W0TU+KigRQ9Z16Numt47hl+kIWf7fL65IkjCjQRQIsLT6OF8Zlk9gimtHTFrBqyx6vS5IwoUAXaQTpCc14cVwOzWMiuXHKfD1RKieFAl2kkXRIas4L47Ixg1FT52vuF2l0CnSRRpSV2pIZtwyi7GAlN02dr1kapVEp0EUa2SntEph680A27TzAzc8upOzgIa9LkhClQBc5CQZ1TuLpG09n1ZY9THg+n/LKw16XJCFIgS5ykpzdM42/XNWHr9aWct+rS6mq0hqlElhaR0vkJLpiQAbFZeX8+f3VpLaM5YHLelE987RIwynQRU6y287MYtueg0z7qpD0hDhuPTPL65IkRCjQRU4yM+OBS3tTXFbOn95bRbvWzbi0T7rXZUkIUKCLeCAiwnjo6r5s232Qe19ZTFqrWAZmat4XaRh9KCrikbjoSCaP8pHRuhm3PpdHQYmeJpWGUaCLeCixRQzTxwwi0owxzy7Ug0fSIAp0EY91TG7O5NE+tu05yPiZ+Rw8pDHqcmIU6CJNwICOiTx8TT/yN+zkN68txat1CiS4KdBFmohL+6Tz6wt78PaSzTzy8Rqvy5EgpFEuIk3I7cO7sH77Ph6bvZYuaS0Z0a+91yVJEKnPmqLTzKzYzJbX0Wa4mS02sxVm9nlgSxQJH2bGn352GoM6J/Hr15aSv2Gn1yVJEKlPl8t04KLaNppZa2Ai8FPn3CnA1YEpTSQ8xURF8PSNp5OeEMdtM/Mo2rnf65IkSBwz0J1zc4AddTS5HnjDObfR3744QLWJhK2kFjFMHT2Q8soqxs3IY195pdclSRAIxIei3YFEM/vMzPLNbFRtDc1svJnlmVleSUlJAA4tErq6prXkyesH8G3xXn7x8mLNzijHFIhAjwJOBy4FLgQeMLPuNTV0zk1yzvmcc77U1NQAHFoktJ3ZPZUHLu3Fxyu38dePvvG6HGniAjHKpQjY7pzbB+wzszlAX0DjrkQCYPSQTL7ZtpeJnxXQvU08l/fXyBepWSDu0N8CzjCzKDNrDmQDqwKwXxGheuTLH0ecQk5WEr95fSmLv9vldUnSRNVn2OIsYB7Qw8yKzGysmU0wswkAzrlVwAfAUmABMMU5V+sQRxE5ftGREUy84XTatIpl/HN5bNtz0OuSpAkyrx4x9vl8Li8vz5NjiwSrb7aWccXEr+ia1pKXbxtMXHSk1yXJSWZm+c45X03b9Oi/SBDp0TaeR67tx5Ki3fz2dc35Iv+XAl0kyFxwSlt+dX53/r54M5PmrPO6HGlCFOgiQejOc7py6Wnp/PmD1Xy6Ws/ySTUFukgQMjMevLoPvdq24u5Zi1hbrNWORIEuErSax0QxebSPmKgIbn0uj90HDnldknhMgS4SxNq3bsbTN51O0c793DVrEYc1PUBYU6CLBLmBmUn8ccSpzFlTwp/f1zN94UwLXIiEgJGDOrJqyx4mf1FIz7atuPL0DK9LEg/oDl0kRDxwWW8GZyXzL28uY9FGLYwRjhToIiGienqAAbRpFcttM/PZulvTA4QbBbpICElsEcOUUQPZV17JbTPzOHjosNclyUmkQBcJMT3axvO36/qzdJOmBwg3CnSREHR+7zbcd0EP/r54M099XuB1OXKSaJSLSIi6fXgXVm8t48EPv6FbWjzn927jdUnSyHSHLhKizIwHr+rDae0T+MVLi1i9dY/XJUkjU6CLhLC46Egm3eSjRWwU42bkUbq33OuSpBEp0EVCXNuEOCaN8lFcVs6E5/Mpr9TIl1ClQBcJA/06tObBq/qwcP1O/v3N5Rr5EqLqs6boNDMrNrM61wk1s4FmdtjMrgpceSISKCP6tefuc7ryan4RU74o9LocaQT1uUOfDlxUVwMziwT+B/gwADWJSCP5xXndueS0tvz3+6v4ZOU2r8uRADtmoDvn5gA7jtHsLuB1QEuniDRhERHGQ1f347T2Cdz90iJWbN7tdUkSQA3uQzez9sDPgKcbXo6INLZmMZFMGeUjoVk042bkUbxHc76EikB8KPo34H7n3DE/Ojez8WaWZ2Z5JSUlATi0iJyItFZxTBntY/eBQ4x7Lo8DFRr5EgoCEeg+4CUzWw9cBUw0s8trauicm+Sc8znnfKmpqQE4tIicqFPaJfDYdf1Ztmk3v3hZqx2FggYHunOus3Mu0zmXCbwG3O6c+3uDKxORRnde7zY8cGlvPlyxTasdhYBjzuViZrOA4UCKmRUB/wFEAzjn1G8uEuTGDM1kQ+k+Jn9RSMfkFtyU08nrkuQEHTPQnXMj67sz59zNDapGRE46M+N3PzmFop0H+I+3ltO+dRzn9NREXsFIT4qKCJERxmMj+9O7XSvufHERy4o0nDEYKdBFBIAWsVFMGz2QxOYxjJm+kO927Pe6JDlOCnQR+UFaqzhm3DKQisrDjH52Abv2V3hdkhwHBbqI/B9d0+KZPMpH0Y4DjJuhdUmDiQJdRH4kOyuZh6/tS/7Gndw9S2PUg4UCXURqdFmfdvzust58tHIbv3tLU+4GA60pKiK1GjO0M9v2lPP05wWkxcdxz3ndvC5J6qBAF5E63X9RD0rKynnkkzUktYzRg0dNmAJdROpkZvz5ytPYtb+C3721nMTm0VzWp53XZUkN1IcuIscUHRnBkzcMwNcpkXtfXsycNZottSlSoItIvcRFRzJl9EC6pLbktpn55G841ro3crIp0EWk3hKaRTNzbDZtE+K4+dmFWvGoiVGgi8hxSY2P5flx2cTHRjFq6gIKSvZ6XZL4KdBF5Li1b92M58dlYwY3TJ7PxlLN+9IUKNBF5IRkpbZk5thsDlYe5vopuWzedcDrksKeAl1ETliv9FbMvCWb3fsPcf3kXC047TEFuog0yGkZCUy/ZRDFZeWMnJxLSVm51yWFLQW6iDTY6Z0SefbmgWzedZDrFeqeUaCLSEBkZyXz7JiBFO08wA1Tctm+V6F+sh0z0M1smpkVm9nyWrbfYGZL/V9zzaxv4MsUkWCQk5XM1Jt9bNyxX3fqHqjPHfp04KI6thcCZznn+gD/CUwKQF0iEqSGdElh2s0D+W7HAa6bNE8flJ5Exwx059wcoNZnfJ1zc51zO/0vc4GMANUmIkFqSJcUpo8ZyJbdB7luUi5bdyvUT4ZA96GPBd6vbaOZjTezPDPLKynR5D4ioSw7K5mZY6tHv1zzzDwtOn0SBCzQzexsqgP9/traOOcmOed8zjlfampqoA4tIk3U6Z2SeH5cNrv2V3DNM/M0TUAjC0igm1kfYAowwjlXGoh9ikho6NehNS+NH0xFZRXXPjOPVVv2eF1SyGpwoJtZR+AN4Cbn3JqGlyQioaZ3u1a8MmEwURERXDcpl/wNO4/9Q3Lc6jNscRYwD+hhZkVmNtbMJpjZBH+T3wHJwEQzW2xmeY1Yr4gEqS6pLXl1wmASm0dz45T5fK5FMgLOvFrJ2+fzubw8Zb9IuCkpK2fUtAWsLS7j4Wv68ZO+Ws7ueJhZvnPOV9M2PSkqIidVanwsL43PoV+H1tz90iKmf1XodUkhQ4EuIifd9ysfnderDb9/ZyV/+WA1XvUWhBIFuoh4Ii46kqduGMDIQR2Z+FkB9726lEOHq7wuK6hFeV2AiISvqMgI/vtnp9K2VRyPfLKGbXsOMvHGAbSKi/a6tKCkO3QR8ZSZcc953Xjwqj7krivl6qfmafWjE6RAF5Em4WpfB2bcMojNuw5w+ZNfsaxot9clBR0Fuog0GUO7pvDaz4cQHRnB1c/M5YPlW7wuKago0EWkSenRNp6/3zGUXumtmPD81zz56VqNgKknBbqINDmp8bHMujWHn/Ztx4MffsMvX1nCwUOHvS6rydMoFxFpkuKiI3n0un50b9OSv360hoKSvUy6yUfbhDivS2uydIcuIk2WmXHnOd2YPMpHQfFefvLEl+Str3W9nbCnQBeRJu/83m14846hNI+J5LpJucyct1796jVQoItIUOjeJp637xjGGd1SeOCtFfz6taXqVz+KAl1EgkZC82imjh7IPed247X8Iq6YOJcNpfu8LqvJUKCLSFCJiDDuPb87z948kE27DnDZ41/y4YqtXpfVJCjQRSQond0zjXfvGkbnlBbcNjOf/3p3JRWV4T25lwJdRIJWh6TmvDphMKMGd2LKl4Vc/fRcvtux3+uyPKNAF5GgFhsVyR9HnMpTNwxg3fZ9XPLYF/xjaXhOGVCfNUWnmVmxmS2vZbuZ2WNmttbMlprZgMCXKSJSt4tPS+e9u8+gS2pL7njxa37z2hL2lVd6XdZJVZ879OnARXVsvxjo5v8aDzzV8LJERI7f910wd57dlVfzi7js8S9ZWrTL67JOmmMGunNuDlDXo1kjgOdctVygtZmlB6pAEZHjER0ZwX0X9mDWrTkcPHSYKybO5fF/fktlGKyGFIg+9PbAd0e8LvK/9yNmNt7M8swsr6SkJACHFhGpWU5WMh/ccyaXnJbOQx+v4epn5rF+e2iPWQ9EoFsN79X4TK5zbpJzzuec86Wmpgbg0CIitUtoHs1jI/vz6HX9KCjey0WPzmH6V4VUVYXmtAGBCPQioMMRrzOAzQHYr4hIQIzo156P7j2LnKxkfv/OSkZOzg3J4Y2BCPS3gVH+0S45wG7nXHiOGRKRJqttQhzP3jyQv1zZhxWb93Dh3+bwbIjdrddn2OIsYB7Qw8yKzGysmU0wswn+Ju8B64C1wGTg9karVkSkAcyMawZ24KN7z2RQ5yT+8M5Krn5mHmuL93pdWkCYV1NQ+nw+l5eX58mxRUScc7y5aBN/fHcl+8sPc8fZXZkwPIvYqEivS6uTmeU753w1bdOToiISlsyMKwZk8PG9Z3HRqW155JM1XPLoFywoDN4FNBToIhLWUuNjeWxkf6aPGUh5ZRXXPDOP+15dQunecq9LO24KdBERYHiPND6690x+PrwLf1+0iXMe+pwX5m/gcBB9aKpAFxHxax4Txf0X9eT9e86gV3o8//bmci5/8ivyN+z0urR6UaCLiBylW5t4Zt2aw6PX9aO47CBXPjWXX72yhOI9B70urU4KdBGRGpgZI/q1Z/avhjPhrC68vWQTw//6GU9+urbJrmWqQBcRqUOL2Ch+e3FPPr73LIZ1TeHBD7/hvIc/5+0lm/Fq2HdtFOgiIvWQmdKCSaN8vDgum/i4aO6etYjLJ85tUsMcFegiIsdhSNcU3r1rGH+9ui/bdh/kmmfmMW5GHmu2lXldmp4UFRE5UQcqDjPtq0Ke/qyAfRWVXDkgg3vO60ZGYvNGO2ZdT4oq0EVEGmjnvgomfraWGXM3AHB9dkduP7sLafFxAT+WAl1E5CTYvOsAj8/+llfyioiJjGDU4E6MPzOL5JaxATuGAl1E5CQq3L6PRz9Zw9tLNhMXHcmowZncekbngAS7Al1ExANri/fy+Oxvq4M9KpIbczpy65lZDeqKUaCLiHhobXEZT35awFuLNxEdGcGvL+zBuDOyTmhfmj5XRMRDXdPieeTafsz+1XBG9GtHRmKzRjlOVKPsVUREfiQzpQV/uapvo+1fd+giIiGiXoFuZheZ2TdmttbMflvD9o5m9qmZLTKzpWZ2SeBLFRGRutRnkehI4EngYqA3MNLMeh/V7N+BV5xz/YHrgImBLlREROpWnzv0QcBa59w651wF8BIw4qg2Dmjl/z4B2By4EkVEpD7qE+jtge+OeF3kf+9IvwduNLMi4D3grpp2ZGbjzSzPzPJKSkpOoFwREalNfQLdanjv6MHrI4HpzrkM4BJgppn9aN/OuUnOOZ9zzpeamnr81YqISK3qE+hFQIcjXmfw4y6VscArAM65eUAckBKIAkVEpH7qE+gLgW5m1tnMYqj+0PPto9psBM4FMLNeVAe6+lRERE6iej367x+G+DcgEpjmnPuTmf0RyHPOve0f9TIZaEl1d8xvnHMfHWOfJcCGE6w7Bdh+gj8bzMLxvMPxnCE8zzsczxmO/7w7Oedq7LP2bC6XhjCzvNrmMghl4Xje4XjOEJ7nHY7nDIE9bz0pKiISIhToIiIhIlgDfZLXBXgkHM87HM8ZwvO8w/GcIYDnHZR96CIi8mPBeocuIiJHUaCLiISIoAv0Y03lGwrMrIN/OuJVZrbCzO7xv76ooH4AAAM7SURBVJ9kZh+b2bf+PxO9rrUxmFmkfyrmd/2vO5vZfP95v+x/wC1kmFlrM3vNzFb7r/ngcLjWZnav/+/3cjObZWZxoXitzWyamRWb2fIj3qvx+lq1x/z5ttTMBhzPsYIq0Os5lW8oqAR+5ZzrBeQAd/jP87fAP51z3YB/+l+HonuAVUe8/h/gEf9576R6qolQ8ijwgXOuJ9CX6nMP6WttZu2BuwGfc+5Uqh9avI7QvNbTgYuOeq+263sx0M3/NR546ngOFFSBTv2m8g16zrktzrmv/d+XUf0PvD3V5zrD32wGcLk3FTYeM8sALgWm+F8bcA7wmr9JSJ23mbUCzgSmAjjnKpxzuwiDa031EpjNzCwKaA5sIQSvtXNuDrDjqLdru74jgOdctVygtZml1/dYwRbo9ZnKN6SYWSbQH5gPtHHObYHq0AfSvKus0fwN+A1Q5X+dDOxyzlX6X4faNc+iet6jZ/3dTFPMrAUhfq2dc5uAv1I9D9QWYDeQT2hf6yPVdn0blHHBFuj1mco3ZJhZS+B14BfOuT1e19PYzOwyoNg5l3/k2zU0DaVrHgUMAJ7yr/i1jxDrXqmJv894BNAZaAe0oLq74WihdK3ro0F/34Mt0OszlW9IMLNoqsP8BefcG/63t33/65f/z2Kv6mskQ4Gfmtl6qrvTzqH6jr21/9dyCL1rXgQUOefm+1+/RnXAh/q1Pg8odM6VOOcOAW8AQwjta32k2q5vgzIu2AK9PlP5Bj1/v/FUYJVz7uEjNr0NjPZ/Pxp462TX1picc//inMtwzmVSfW1nO+duAD4FrvI3C6nzds5tBb4zsx7+t84FVhLi15rqrpYcM2vu//v+/XmH7LU+Sm3X921glH+0Sw6w+/uumXpxzgXVF9UrIq0BCoB/87qeRjrHYVT/mrUUWOz/uoTq/uR/At/6/0zyutZG/G8wHHjX/30WsABYC7wKxHpdX4DPtR+Q57/efwcSw+FaA38AVgPLgZlAbChea2AW1Z8THKL6DnxsbdeX6i6XJ/35tozqUUD1PpYe/RcRCRHB1uUiIiK1UKCLiIQIBbqISIhQoIuIhAgFuohIiFCgi4iECAW6iEiI+F8BbO5CtVENywAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(loss_scores2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXwU9f3H8dcnNzkgBAKEkECAcN8E8KSCYAEteFTL4W21h2jVn7VWrfXqpbXWtrQVFESr4oEiYgRUvA9MAnIFAuFMwpGQi9ybzX5/f8yqMSRkQ3az2c3n+XikZGa+O/MZB98dv/ud74gxBqWUUr4vwNsFKKWUcg8NdKWU8hMa6Eop5Sc00JVSyk9ooCullJ8I8taBu3fvbvr16+etwyullE/KyMg4boyJbWyb1wK9X79+pKene+vwSinlk0TkYFPbtMtFKaX8hAa6Ukr5CQ10pZTyExroSinlJzTQlVLKT2igK6WUn9BAV0opP+G1cehKKeU1O9+CI1tP++N2h4PqWgeVtXVU2+qoqq2j0lZHbZ2DLp2C6RYZQuewYE5U11JUbqO4spb6U5XHjJvDoHE/cMeZfI8GulKq46izw/p7YeN/nSuk2Y+YRn4PMBCO9XMqMc6fhtI6x4EGulJKnaaqYnj1Otj3AZy5EKY9CIFWBJZU2tiTX86eY+XklVRSVGHjeLmN7Pxy9h+vACA0KICBPSIZ1DOKft0i6B4VQreIEGIiQomJsH4PCw7kQGEFe/LLyS2uJD66E4N6RpHUPYKw4MBvS5nkoVPUQFdKeVddLay7FzYtB+Nw++6N83+MqcOBsKTLHTz91VmUffyuc7uhtu67+/DAAKFruBXQA2IjmDcxgTP6d2NYXGeCApv/2nFoXGeGxnV2+3m4QgNdKeU9lUXwytVw4BMY9RPo3Pu0d2UMFFbYOFpaRXmNnSpbHZW1deSfqKGqtg4DfBJ0BrVhY5meGEnXiJBvPxvdKZhBPaNI7hlJ7y6dCAhoviumPdJAV0qdmqMOtq+EiuPu3a9xQNoSOHEELnkKRs9t/iPGcKS0mt3HythXUMHx8hqKKmwcO1HNpkMllFbVAhAUIHR1doMMHdKZM/rHMCmpG7/sFo6Ib4a1KzTQlVJNqz4BK2+APes9s/+oOGqueovj0aMoP1rWaJMT1bV8tb+IjfuL2HywmLIa+7fb6gf3jOG9mNQ/holJMcRHd/Lr4G6KBrpSqnGFe+GleVC0Fy58HEb82C27rTOGz7KP83JaLp8erKD0v8eBDc1+LrlHJLPH9GZoXGeSe0QyoEckMeEhPts94gka6EqpkxUfgKfPBwSuWgVJ57Z4F7V1Dg4WVjhHjlRRVGGjqMLGp9nHyS2uIjYqlDnj+9GzcxjdIkKICgumsZvq0KAAxiRE0y0ytNWn5e800JVS32cMpN5ljT752cfQbUCjzQrKakg/UEReSRWFFTaKym0UVtRYv1fYyCuuwu74bvTIN90jg3tGcc+soUwf1pNgF0aNKNdpoCulvi8rFfasgwv+8L0wt9c52Li/iLXbj/L53uPsLaj4dltwoBDjHJPdLSKEhK7hzBwRx6CekST3iCIxJpzOnYI6ZL92W9JAV0p9x1YJ79wNPYbBpJ9xtLSajfsL+Sz7OO9mHqO4spZOwYGc0T+Gy1MSmJQUw4AekUSFali3BxroSinA6vM+sPIBkksP8WjcE6x5/FMOFVUCEBUaxJQhPZg1shc/GNSDTiGBzexNeYNLgS4iM4AngUDgaWPMnxtsTwSWA9HONncbY1LdXKtSykPySqp46tll3Ff8NK87zmV9xQCG947k6jP7MimpG8N6dyZQR5O0e80GuogEAouA6UAukCYiq40xmfWa3Qe8Yoz5j4gMA1KBfh6oVynlZut3HOWrVx/jd2YZVZ37cdFPn+HSLj29XZY6Da7coU8Eso0x+wBEZAUwB6gf6Ab4ZvKCLsBhdxaplHJRRSHkfuVS00qbnVczcgnOXs99QRuoTJpG57nLIMw785Co1nMl0OOBnHrLuZw8WdgDwHoRuQWIAKY1tiMRuQm4CSAxMbGltSqlTiU3HVbMh/JjLjUPB64BCAL7Wb8ifNrvIUD7xn2ZK4HeWMeZabA8D3jWGPO4iJwJPC8iI4z5/tRpxpjFwGKAlJSUhvtQSp2uLS/D6lugc5z1IFCn6JOa1NYZvtxXSOq2I2zLKyUxJpxbz09mSN94gpoYa658iyuBngsk1Fvuw8ldKjcAMwCMMV+ISBjQHch3R5FKqXrsNvjgETi00Vp21EJeBvQ7F654DsK/e6WCw2HYnFPCO9uO8OaWwxSU2ejTtQ9XzTiba8/uR2iQ3pH7E1cCPQ1IFpEkIA+YC8xv0OYQcD7wrIgMBcKAAncWqpTCmvHwlWvg4KeQeCYEhgChcM4dMOUeCAzG4TBkHComddsR1m4/ypHSakICA5g8KJYFkxKZPChWR6z4qWYD3RhjF5GFwDqsIYlLjTE7ROQhIN0Ysxr4P2CJiNyO1R1zran/Aj2lVOsd2wEvzYWyY3DpEhh1xbeb6hyGtANFvLPtCO9sP0p+WQ0hQQFMTo7lrhmDOX9oTzqHBXuxeNUWXBqH7hxTntpg3f31fs8EznZvaUqpb+16G1beCKFRcN070Gc8AOU1dv774V5WpOVwvLyG0KAAzhscy6yRcUwd0oMoDfEORZ8UVao9MwY+eRw2PAy9x8LcF6FzbxwOw+ub8/jL2l0UlNVwwbCezB7TmymDexARqv9ad1R65ZVqTxwOeP9BKNhlLVcWQm4ajLwcZv+Tckcwq748yP++PMiuo2WMSYhm8VXjGZvY1bt1q3ZBA12p9mTTcvjs7xA7FIJCqHXA/tG/4f3oK8hauYv3duZTXmNnWFxnnvjJaOaMjtcXPKhvaaAr1V5UFFp3533Ppnzem/zrg70s/XQ/toMOIItencO4YHhPrjyjL2MTonV2Q3USDXSl2ov3fo+pKWNd31/zu8c/oqCshsvG9WH+pESSe0bqKBXVLA10pdoB+4EvCNr8PCuCL+a36ysZmxjNkqtTGJNw8hOfSjVFA10pL8vMLSLkuV8QYWJYGbWAxRePYvqwntqlolpMA10pL1q7/SibXvkT9wTsZ+tZ/+DVC6ZrkKvTpoGulBcYY/jXhmyee3cjH3V6mZrE8xh1wdU0+tp7pVykga5UGzPG8NCaTJZ9doCVPd6gU0UdMvtvGuaq1QK8XYBSHYkxhgdW72DZZwd4cFQh40+8h5x9G+j0tcoNNNCVaiMOh+H+N3ew/IuD/PLMHlxd9C+I7gvn3uHt0pSf0C4XpdqAw2H43ZvbeWHjIe6aFMYv8m5FCrNh/isQ3Mnb5Sk/oYGulIc57LXcv2oLr6bn8qex5czd/TvE1MGVK2HAFG+Xp/yIBrpSHuTYswH7i/N4xFTzSBiwE+g+COat0H5z5XYa6Ep5SE1VOSdeuZnyui7kJl3POQO6IyHhMGY+hHXxdnnKD2mgK+UBZdW1rP/3b7is9jBfjvsvP7p4nrdLUh2AS6NcRGSGiGSJSLaI3N3I9idE5Gvnz24RKXF/qUr5hqIKG3f893UuOvEyh+JnaZirNtPsHbqIBAKLgOlALpAmIqudr50DwBhze732twBjPVCrUu1edW0dNzz7FbcXLSIwJITEuU94uyTVgbhyhz4RyDbG7DPG2IAVwJxTtJ8HvOSO4pTyJQ6H4fYVmznzyHNMDthC0LTfQVQvb5elOhBX+tDjgZx6y7nApMYaikhfIAnY0MT2m4CbABITE1tUqFLt3WOpW5i2+/dcFvQpDL8EJtzo7ZJUB+NKoDc2wYRpou1c4DVjTF1jG40xi4HFACkpKU3tQymfUlxh49+pXzJr2+2MDczGnHcP8oO7dG4W1eZcCfRcIKHech/gcBNt5wI3t7YopXyBvc7B/748yBPv7eEvdY8xMvAQdT9eTuDwi71dmuqgXAn0NCBZRJKAPKzQnt+wkYgMBroCX7i1QqXaodo6B798YRPvZh7jl/H7mFH4FUz5HWiYKy9q9ktRY4wdWAisw3rO7RVjzA4ReUhEZtdrOg9YYYzRrhTl1xwOw12vbeXdzGM8NGsAv3Y8A92S4axbvF2a6uBcerDIGJMKpDZYd3+D5QfcV5ZS7ZMxhgff2sEbm/O484JBXO1YCcX74eo3ISjU2+WpDk6nz1WqBf61IZvlXxzkxnOTuHl0IHzyNxhxGfQ/z9ulKaWBrpSr1u04yuPv7uaSsfHcM2so8snjEBAEF/zB26UpBWigK+WS3cfKuOPlrxndpwt/unQkUl0C21fCqCugc5y3y1MK0EBXqlmllbXc9Fw6nUKC+O9V4wkLDoQtL4O9ClKu83Z5Sn1LA12pU6iosXPj8+nklVTx1FXjiOvSCYyBjGUQPx7iRnu7RKW+pYGuVBPKa+xctyyN9ANFPH7FGMb3jbE2HPoCCnbBeL07V+2LzoeuVCPKqmu5blkam3NKeHLuWH40uvd3G9OXQWgXGHGp9wpUqhF6h65UA2XVtVyz9Cs255Twj4ZhXlEImatg9E8gJMJ7RSrVCL1DV6qeE9W1XP3MV2zPK2XR/LHMGBFnfQG6abnVoLII6mza3aLaJQ10pZxKq2q5eulXZB4uZdGCcfxweC/rC9CP/gy2CuvlzhHdYeBC6DnM2+UqdRINdKWAfQXl/PKFTewtKOc/C8YzbVhPa8PxPVC0D2b9FSbq/OaqfdNAVx3eG5tzufeN7YQGBfDMNROYPCj2u41ZzimMBs/0TnFKtYAGuuqwaux13L9qBy+n5zCxXwxPzhtjjTOvb/da6DUSuvTxTpFKtYAGuuqQ8suq+fnzGWw6VMLNUwZw+7RBBAU2GPRVUQg5G2Hyr71TpFItpIGuOpzteaXc+Fw6xZU2/r1gHLNGNjEXy571YBza3aJ8hga66lA27ivkumfTiO4UzGs/P4sR8V2abpyVClFxEDem7QpUqhVcerBIRGaISJaIZIvI3U20uUJEMkVkh4i86N4ylWq9L/YWcu2yNOK6hLHq5rNPHeb2Gti7AQbN0Jc9K5/R7B26iAQCi4DpWC+MThOR1caYzHptkoHfAmcbY4pFpIenClbqdHyefZzrl6eR0DWcF288g9ioZt4udOATsJXD4FltU6BSbuBKl8tEINsYsw9ARFYAc4DMem1uBBYZY4oBjDH57i5UqdNhr3Pw1Mf7ePK9PSR1j+CFGyfRPbKRMDcG0p6Gw19by0e3QnA4JE1u24KVagVXAj0eyKm3nAtMatBmEICIfAYEAg8YY9Y23JGI3ATcBJCYmHg69Srlsp1HTvDr17awPe8Es0b24pGLRxITEXJyQ1slrF5ovbAisicEBFvrJ/wUgsPatmilWsGVQG+sA9E0sp9k4DygD/CJiIwwxpR870PGLAYWA6SkpDTch1Jus2brYe54eQtRYUGnHslSmgcr5sORLTDtATj7Nu0zVz7LlUDPBRLqLfcBDjfS5ktjTC2wX0SysAI+zS1VKtUCz362nwfXZJLStytPXZXS+F05WHfmz86CiuMw7yUdnqh8niujXNKAZBFJEpEQYC6wukGbVcAUABHpjtUFs8+dhSrVHGMMf12XxQNvZTJtaE+ev2FS02EO8MlfofgAzFuhYa78QrN36MYYu4gsBNZh9Y8vNcbsEJGHgHRjzGrntgtEJBOoA35tjCn0ZOFK1Vdlq+OulVt5a8th5k5I4JGLR5z85Gd9x/fAZ/+A0fMg6dy2K1QpDxJjvNOVnZKSYtLT071ybOVfjpZWc+Nz6Ww/XMpdPxzCz3/QHzlVP7gx8Nwca0TLLekQqaNsle8QkQxjTEpj2/RJUeXTPtpdwJ2vbqGyxs6Sq1K+m/b2VHa8Dvs/sqbE1TBXfkQDXfmk0qpa/vB2Jq+k5zIgNoL/3TCJwb2imv9gbgak3mU9zp9yvecLVaoNaaArn7Mlp4SfPZ9Bflk1vzhvAL86P5mw4MDmP7j1FXhzIUT1hEuXQIALn1HKh2igK5+y+VAxVz/zFdERway6+WxG9Yl27YPvP2yNaul7DlzxHER082yhSnmBBrryGRkHi7lm6Vd0iwzhpRvPoHd0p+Y/BLAr1QrzsVfChU9A0CmGMirlwzTQVbtXVl3L65vyeGxdFrFRobx446ST3yzUFFslvPMbiB0KF/0dAoM9W6xSXqSBrtolh8OwOaeY1zLyePPrPCptdYxLjObfC8bTq0sL5lf55HEoPQTXpmqYK7+nga7alUOFlSz9bD9rtx/l6IlqwoIDmD26N1ee0df1/vJvHN8Dn/8DRs2Ffmd7pmCl2hENdNVufLKngJtf2ES13cF5g2K5e+QQzh/ag6iw07izdjgg9U4I6gQXPOz+YpVqhzTQldcZY3j28wM88vZOkntEsuTqFBJiwk9/hzVl8PrPYN+HcOHj+vCQ6jA00JVXlVTaeGD1DlZ9fZjpw3ryxE/GEBnair+WxQfgpXlQkAUzH4WUG9xWq1LtnQa68pq1249y36rtlFTauH3aIG6ZOpCAgFbMRV58EJZMBYcdrlwJA6a4r1ilfIAGumpz5TV27ntjG6u+PsywuM4sv34Cw3uf4oXNrnrnN1BbDT/7CLont35/SvkYDXTVprbnlbLwxU0cKqrkV+cns3DqQIJPNc2tq7Legd3vwPSHNMxVh6WBrtpEbZ2D5Z8f4NG1WcREWE96TurvpsfvbZXwzl0QOwTO+KV79qmUD9JAVx730e4CHl6TSXZ+OecP6cFjl48+9ZuEWurTv0HJIbj2bX14SHVoLgW6iMwAnsR6Y9HTxpg/N9h+LfAYkOdc9S9jzNNurFP5oKOl1fzuze28m3mMvt3CWXzVeKYP63nql0+4Ys+78OGfoa7GWs7fBaN+Av3OaX3RSvmwZgNdRAKBRcB0rJdBp4nIamNMZoOmLxtjFnqgRuVjjDG8mpHLw2syqa1z8JsZQ7j+nH6EBrVyulpj4PN/wrv3Q7cB0H2Qtb7XKJj2YOsLV8rHuXKHPhHINsbsAxCRFcAcoGGgqw7MZnewNbeEjfuLeH/nMTYdKmFiUgyPXjaKft0jWn+A2mpYcxtseQmGXQwX/xtC3LBfpfyIK4EeD+TUW84FJjXS7jIRmQzsBm43xuQ0bCAiNwE3ASQmJra8WtXuGGNYveUwD76VSVGFDYDBPaN4aM5wrpzUt3Xjyr9RdgxeXgC5aXDePfCDu6C13TZK+SFXAr2xf3Mavln6LeAlY0yNiPwcWA5MPelDxiwGFoP1kugW1qramfwT1dy7yuojH50QzR8vGcHEpG7u/cLz8NewYj5UFVsvphg2x337VsrPuBLouUBCveU+wOH6DYwxhfUWlwB/aX1pqr0qraxl6Wf7Wfrpfmx1Du6dNZTrz0ki0B134/Xt+whe/AlEdIfr10HcKPfuXyk/40qgpwHJIpKENYplLjC/fgMRiTPGHHEuzgZ2urVK1S4UV9h45tP9PPv5Acpr7MwY3ou7Zgymf2yk+w9mq4TVCyE6wZrLPDLW/cdQys80G+jGGLuILATWYQ1bXGqM2SEiDwHpxpjVwK0iMhuwA0XAtR6sWbWxogobSz7Zx3OfH6DCVseskb24ZWoyQ+M6e+6gnzz+3dhyDXOlXCLGeKcrOyUlxaSnp3vl2Kp5VbY6PsjKJ3XbEd7fmU+1vY4LR8Zx6/nJDOoZ5dmDH8+G/5wJwy+BSxd79lhK+RgRyTDGpDS2TZ8UVSd58+s8fvv6NiptdXSLCOGScfFcd1Y/kj0d5GCNNU/9P+vFFNP1xRRKtYQGuvqepZ/u56E1mUzo15Xbpw1iYlIMQe6YPKuhHW/A9tdPXl9bab2YYuZjENXT/cdVyo9poCsAqmvrePL9Pfznw73MGN6Lv88dQ1hwK5/sbIox8O7voboUouJO3j5qLkzQF1Mo1VIa6B3UsRPVrN1+lE+zj5OdX87BwgocBuZPSuThOSPcPwSxvvydUHIQfvQkjL/Wc8dRqoPRQPdTNruD0qpauoYHExQYgM3uYFteCV/uK+LDrHzSDxZjDPTrFs6w3p350ejejO7ThalDerR+8qzmZKVafw6a4dnjKNXBaKD7mUOFlbyw8SCvpOdQXFkLQHR4MNW1dVTXOgAY0iuK284fxKyRvdrmi86Gdq+F3uMgqlfbH1spP6aB7geOl9ewbsdRUrcd4fO9hQSIMH1oT87oH0NJVS2F5TaCAwOYmNTV/Y/mt1TZMchNhyn3eq8GpfyUBroPqrLVkXGwmI37C/libyGbDhXjMJDUPYJbpyYzb2IivbqEebvMxu1ZBxgYrN0tSrmbBrqP2XSomJ89n0FBWQ2BAcKI+C7cPGUgs0bGMaRXlOf7v1vKUQcS8N3siFlroUsC9Bzh3bqU8kMa6D5kZUYuv319G3HRYSy9NoWJSd2IDG3Hl/DIVmva28594IrlEBoFezfA2Ct1+lulPKAdp4H6RqXNzuPrd/PMp/s5a0A3Fs0fR1dv9oO7InM1vPEzCO0MhzfD4imQci3Yq2DwTG9Xp5Rf0kBvx755ecSf39nFkdJqrjmzL/ddNIxgTzy52Vrpy6D4gPV7xXH4+n8QnwJzX4Cyo9ac5hsegZAoffenUh6igd4O5Z+oZu2Oo6zMyGVLbikj4jvzj3ljmdAvxtulNe7oduv1cAHB3/WXj70SZj0OwWHW8MQbP4BVP4eewyEo1NsVK+WXNNC9yOEwrNtxlBVpOVTZ6gCorLWz4/AJjIGBPSL5y2Uj+fH4BM8+udlaGcsgMBT+bxeEN/F/OlE94ao32rYupToYDXQvcDgMqduP8M/3s8k6VkZCTCf6RIcDEN0pxLsP/bRUTTlsedma6rapMFdKtQkN9Da26VAxD76VyZacEgb2iOTJuWO4aFTv9n0HfirbV4KtDFKu93YlSnV4Guht5GhpNY+u3cXrm/PoERXK45eP5uKx8b4b5N9IXwo9hkHCRG9XolSH51Kgi8gM4EmsV9A9bYz5cxPtfgy8CkwwxujriLCmpV3y8T7+/eFe6ozh5ikD+OV5A4loz+PHXZW3CY58DbP+quPKlWoHmk0VEQkEFgHTgVwgTURWG2MyG7SLAm4FNnqiUF9zqLCSd7Yf4fkvD5JbXMXMEb24Z9ZQEmLCvV2a+2Qsg+BwGHWFtytRSuHaHfpEINsYsw9ARFYAc4DMBu0eBh4F7nRrhT7mw6x8/ro+i+15JwAYkxDNo5eN4qyB3b1c2SkYAxufsl7MXGdz/XM1J2DMAgjr4rnalFIucyXQ44Gcesu5wKT6DURkLJBgjFkjIk0GuojcBNwEkJiY2PJq2zFjDEs+2cef3tlFUvcI7pk1hJkj4tr/HbndBm/fAZufh6TJEDvU9c8GBMEZP/dcbUqpFnEl0BvrHDXfbhQJAJ4Arm1uR8aYxcBigJSUFNNM83bNXucgt7gKsP5h/GtDNis35TJrZC/+evlowkN8oI+8ssh6gvPQFzD513DePRDQDp9CVUq5xJXUyQUS6i33AQ7XW44CRgAfOmf66wWsFpHZ/vrFaE5RJTc+l86uo2XfW3/btGRunZpMgK+MXFlzm/XF5o+XwojLvF2NUqqVXAn0NCBZRJKAPGAuMP+bjcaYUuDbDmIR+RC401/DfOO+Qn7xwibsdQ4emjOcqDDrH2FiTATj+3b1cnUtkP0eZL4JU+/TMFfKTzQb6MYYu4gsBNZhDVtcaozZISIPAenGmNWeLrK9eDU9h9++vo3EmHCeviaF/rGR3i7p9NRWQ+qvodtAOOtWb1ejlHITlzp6jTGpQGqDdfc30fa81pfV/vzvy4Pct2o75wzszqIF4+jSKdjbJZ2+z/8BRfusuVV0oiyl/IYPfHPnfc99cYD739zB1CE9+M+V4wgNCvR2Sc0rzYPctJPX26ut4YnDLoYBU9u+LqWUx2igN+OFjQe5/80dTBvag0ULfCTM926AV6+F6tLGt4dFww//2KYlKaU8TwP9FI6X1/DImp2cm9ydfy8YT0hQOx/S980DQuvugdjBsOA1CGmkn79zHHTyoS9wlVIu0UA/hac+2kuNvY4HZw9vv2G+4Q+Q/a71u70G8jNh8IVw6VPWOzyVUh2GBnoT8suqef7Lg1w8Jr79jmbZ8x58/CjEj4fwbta6kZfD2bfpA0JKdUAa6E3474f7qK0z3HJ+srdLaVxtNaTeaQ09vO4dHa2ilNJAb0z+iWpe2HiQS8bGk9Q9wtvlNO6zJ6F4P1y1SsNcKQWA/nd5I/794V7sDsOtU9vp3XnRfvj0bzD8UhgwxdvVKKXaCb1DbyCnqJIXNx7i8vF9SOzWTmZKrC6Fd38PJQet5aL91kyHP/yDd+tSSrUrGugNPLoui4AAuG3aIG+XYincCy/NtZ7sjBtjvRkoIhamPQCde3u7OqVUO6KBXs/XOSW8teUwt0wdSK8uYd4p4sRhqCiwfi8+AKtvBQmw+sqTzvVOTUopn6CB7mSM4Y9v76R7ZAg/+8EA7xSRkwbLZoKj9rt1PYbB3BchJsk7NSmlfIYGutO7mcf46kARD188gkhvvMDZUWe9OSgiFmY9ZnWtBARBv3MgpJ2OtFFKtSsa6Fh354+uy6J/bARzJyQ0/4HWqiyCT5+ACT+Frn2tdWnPwNGtcPmzMPQiz9eglPI7OmwR+HJfEdn55dx83kCCA9vgH8lXS6wpbJdMgQOfQdkx2PAw9J9izYKolFKnQe/QgZfTDhEVFsSFo+I8f7A6O2xaDr3HQc0JeG4O9BxmTWs7669WV4tSSp0Gl25HRWSGiGSJSLaI3N3I9p+LyDYR+VpEPhWRYe4v1TNKKm2kbj/KJWPjCQtug6lxs9+FE3lwzu3w0/chaTIc2QJn/wq6D/T88ZVSfqvZO3QRCQQWAdOxXhidJiKrjTGZ9Zq9aIz5r7P9bOBvwAwP1Ot2qzbnYbM7+Elb9J0DpC+DyF4weCYEBsP8V+DAx9BPhyQqpVrHlS6XiUC2MWYfgIisAOYA3wa6MeZEvfYRgHFnkZ5ijGFFWg4j47swvHcX9+y0cK81J7nDbi1HdLfuvkMioOQQ7FkPk++0whwgMEjfHKSUcgtXAj0eyKm3nAtMathIRG4G7gBCgEYTSkRuAm4CSExMbGmtbrc1t5RdR0UEhbMAAA14SURBVMv4wyUj3LPDOju8cjUc3w2hna11lYWQlQpzX4JNz1l95OOucc/xlFKqHlcCvbFv6U66AzfGLAIWich84D7gpNQyxiwGFgOkpKR4/S5+RVoOnYIDmT36NB6hN87y63+JmbYEjm2Hy5fDcOdolT3vwmvXWyNajAMGTofoNureUUp1KK58KZoL1E+gPsDhU7RfAbT7sXdVtjre2nKYWSPjiAoLbvkOVt8C/xgLR7dby2VHrbcHDZgKw+Z81y55Ovz0PevtQZWFkHKde05AKaUacOUOPQ1IFpEkIA+YC8yv30BEko0xe5yLFwJ7aOc+zMqnvMbOZePiW/7hfR/C5uchIBieuQAuXQyZq6CupvGhh7GD4cYN1pjzQT7xXbFSygc1G+jGGLuILATWAYHAUmPMDhF5CEg3xqwGForINKAWKKaR7pb2Zs22I3SPDGFiUszJG3evg04xkDDh5G12G6T+Grr2sybMWnkDvLzA2jb5LujWxDwwnbrqE6BKKY9y6cEiY0wqkNpg3f31fv+Vm+vyqEqbnQ0787lsfDxBDZ8MPbYDXppn/T7zLzDxxu9v/+Jf1pee81+1Jsy69m14+04o2Ann3tE2J6CUUo3okE+KbtiVT1VtHReNavBlqDFWOId1sV68nHon5GfCzEetYYYlh+CjR2HIRTDoAuszwZ3g4kVtfxJKKdVAhwz0t7ceITYqlAn9GnS3bFkBhz6H2f+EMQvg/Qetd3dmPAuINUoluBPM+JM3ylZKqVPqcIFeXmNnw6585k5IIDCg3peXVcWw/j7oMxHGXAkBATD9IUg8C3LTvms3YCpEe38MvVJKNdThAv39nceosTu4cFRvyFz93Xs6D3wGVUVw4RtWmH9j8AzrRyml2rkOF+hvbz1Cz86hpEQUwPKrvr/x3DshbpR3ClNKqVbqUIFeWlnLh7sLWDApkYBNz1rjyG/JgPAYQCA00tslKqXUaetQL7h44auD2OwOrhjdHb5+0RoX3rWv9RSnhrlSysd1mEC32R0s//wA5wzsztCiDVBdAinXe7sspZRymw4T6G9tOcyxEzX89NwkyFgG3QbqHORKKb/SIQLdGMOST/YxqGckP+iSDzkbYfx1+ro3pZRf6RCB/ll2IbuOlvHTc/ojGc9CYCiMmd/s55RSypd0iEBf8sk+ukeGMmdoBGx92ZqrPLyRSbmUUsqH+X2g7zhcyke7C7jmzL6EfvRHsJXDmQu9XZZSSrmdXwe6MYY/pu6ka3gw1/YvhfRnYMJP9eEhpZRf8utA/3B3AZ9lF3LLlAFEvXcXhHeHKfd6uyyllPIIv31StM5h+HPqLvp2C+fq0I8gLwMuWQydor1dmlJKeYRLd+giMkNEskQkW0TubmT7HSKSKSJbReR9Eenr/lJb5rWMHLKOlXHP1HiCPngI+p4Do67wdllKKeUxzQa6iAQCi4CZwDBgnogMa9BsM5BijBkFvAY86u5CW6LKVsfj63czNjGaC8KzrKlxz7tbx50rpfyaK3foE4FsY8w+Y4wNWAHMqd/AGPOBMabSufgl0Me9ZbbMezuPkV9Ww50XDEYOfmGNO0+Y6M2SlFLK41wJ9Hggp95yrnNdU24A3mlNUa31QVY+0eHBnNG/Gxz8DPpMgKBQb5aklFIe50qgN9ZPYRptKHIlkAI81sT2m0QkXUTSCwoKXK+yBRwOw0dZBfxgUCyBtjI4uhX6ne2RYymlVHviSqDnAgn1lvsAhxs2EpFpwL3AbGNMTWM7MsYsNsakGGNSYmNjT6feZm3LK6WwwsaUwT0g5yvrPaB9z/LIsZRSqj1xJdDTgGQRSRKREGAusLp+AxEZCzyFFeb57i/TdR9k5SMCkwfFWt0tAUFWl4tSSvm5ZgPdGGMHFgLrgJ3AK8aYHSLykIjMdjZ7DIgEXhWRr0VkdRO787gPsgoYmxBNTEQIHPwceo+FkAhvlaOUUm3GpQeLjDGpQGqDdffX+32am+s6LcfLa9iaW8Id0wZBbZX1MNGZv/R2WUop1Sb86tH/j3cXYAxMGdIDctPAUQt99QtRpVTH4FeB/kFWAbFRoQyL62x1tyCQMMnbZSmlVJvwm0C31zn4eHcB5w2KJSBArC9Ee43QuVuUUh2G3wT6pkMllFbVWt0tdhvkpGl3i1KqQ/GbQH9jcx6dggOt4Yo5X4K9SgNdKdWh+EWgV9nqWLPlMDNH9iIyNAg2PQehXWBguxh8o5RSbcIvAn3djqOU1di5fHwCVBRC5pswei6EhHu7NKWUajN+EeivZuSQENOJSUkx8PULUGeDlOu8XZZSSrUpnw/03OJKPt9byI/HJRCAgYxnIeEM6DHU26UppVSb8vlAX5mRB8Bl4+PhwMdQtBdSrvdyVUop1fZ8OtAdDsNrm3I4a0A3+nQNh/Rl0KkrDJvT/IeVUsrP+HSgf7m/kJyiKuvL0NI82LUGxiyA4DBvl6aUUm3OpwP9qY/2ERMRwozYQlg6w5oqV7tblFIdlM8G+tbcEj7aXcAfhuwnbPkMayKu61Kh2wBvl6aUUl7h0vS57dG/NmQzIyyTmTsegfjx8JMXoHOct8tSSimv8clA33nkBOszj/FOQgaUx8C1b0NwJ2+XpZRSXuWTXS6LPsimS6gwuOwLGPRDDXOllMLFQBeRGSKSJSLZInJ3I9sni8gmEbGLyI/dX+Z39haU8/a2I/xmeAkB1SUweKYnD6eUUj6j2UAXkUBgETATGAbME5FhDZodAq4FXnR3gQ2t2XKE0KAALg7fCoEhMGCqpw+plFI+wZU79IlAtjFmnzHGBqwAvvfkjjHmgDFmK+DwQI3fc+v5A1n7q8mE71sP/c6F0ChPH1IppXyCK4EeD+TUW851rmsxEblJRNJFJL2goOB0doGI0I/D1iP+2t2ilFLfciXQpZF15nQOZoxZbIxJMcakxMbGns4uLFmp1p+DZpz+PpRSys+4Eui5QEK95T7AYc+U46KstdBrJEQnNN9WKaU6CFcCPQ1IFpEkEQkB5gKrPVvWKVQUWq+YGzzLayUopVR71GygG2PswEJgHbATeMUYs0NEHhKR2QAiMkFEcoHLgadEZIfHKt6zHoxDu1uUUqoBl54UNcakAqkN1t1f7/c0rK4YzwvrAoMvhLgxbXI4pZTyFb736P+QWdaPUkqp7/HJR/+VUkqdTANdKaX8hAa6Ukr5CQ10pZTyExroSinlJzTQlVLKT2igK6WUn9BAV0opPyHGnNbEia0/sEgBcPA0P94dOO7GcnxFRzzvjnjO0DHPuyOeM7T8vPsaYxqdrtZrgd4aIpJujEnxdh1trSOed0c8Z+iY590Rzxnce97a5aKUUn5CA10ppfyErwb6Ym8X4CUd8bw74jlDxzzvjnjO4Mbz9sk+dKWUUifz1Tt0pZRSDWigK6WUn/C5QBeRGSKSJSLZInK3t+vxBBFJEJEPRGSniOwQkV8518eIyLsissf5Z1dv1+puIhIoIptFZI1zOUlENjrP+WXne239iohEi8hrIrLLec3P7CDX+nbn3+/tIvKSiIT52/UWkaUiki8i2+uta/TaiuUfzmzbKiLjWno8nwp0EQkEFgEzgWHAPBEZ5t2qPMIO/J8xZihwBnCz8zzvBt43xiQD7zuX/c2vsN5d+42/AE84z7kYuMErVXnWk8BaY8wQYDTW+fv1tRaReOBWIMUYMwIIxHoBvb9d72eBhi9AburazgSSnT83Af9p6cF8KtCBiUC2MWafMcYGrADmeLkmtzPGHDHGbHL+Xob1L3g81rkudzZbDlzsnQo9Q0T6ABcCTzuXBZgKvOZs4o/n3BmYDDwDYIyxGWNK8PNr7RQEdBKRICAcOIKfXW9jzMdAUYPVTV3bOcBzxvIlEC0icS05nq8FejyQU28517nOb4lIP2AssBHoaYw5AlboAz28V5lH/B24C3A4l7sBJcYYu3PZH693f6AAWObsanpaRCLw82ttjMkD/gocwgryUiAD/7/e0PS1bXW++VqgSyPr/HbcpYhEAiuB24wxJ7xdjyeJyEVAvjEmo/7qRpr62/UOAsYB/zHGjAUq8LPulcY4+43nAElAbyACq8uhIX+73qfS6r/vvhbouUBCveU+wGEv1eJRIhKMFeYvGGNed64+9s1/gjn/zPdWfR5wNjBbRA5gdaVNxbpjj3b+Jzn45/XOBXKNMRudy69hBbw/X2uAacB+Y0yBMaYWeB04C/+/3tD0tW11vvlaoKcByc5vwkOwvkRZ7eWa3M7Zd/wMsNMY87d6m1YD1zh/vwZ4s61r8xRjzG+NMX2MMf2wrusGY8wC4APgx85mfnXOAMaYo0COiAx2rjofyMSPr7XTIeAMEQl3/n3/5rz9+no7NXVtVwNXO0e7nAGUftM14zJjjE/9ALOA3cBe4F5v1+OhczwH6z+1tgJfO39mYfUpvw/scf4Z4+1aPXT+5wFrnL/3B74CsoFXgVBv1+eB8x0DpDuv9yqga0e41sCDwC5gO/A8EOpv1xt4Ces7glqsO/Abmrq2WF0ui5zZtg1rBFCLjqeP/iullJ/wtS4XpZRSTdBAV0opP6GBrpRSfkIDXSml/IQGulJK+QkNdKWU8hMa6Eop5Sf+H8qFUA5ETxc9AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(train_scores2)\n",
    "plt.plot(test_scores2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "为了制造出过拟合和L2抑制过拟合的效果，将中间层Unit个数改成300，但是net和net2表现仍一样好。  \n",
    "把训练样本数减少到100，net和net2又表现得一样差。  \n",
    "看上去有没有正则化并没有太大区别。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
